author | Philippe Pepiot <philippe.pepiot@logilab.fr> |
Tue, 21 Feb 2017 10:12:47 +0100 | |
branch | 3.23 |
changeset 11961 | a25e52cd8be4 |
parent 11360 | 49aca289134f |
child 11706 | 43c5b3f31a3a |
permissions | -rw-r--r-- |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
1 |
# -*- coding: utf-8 -*- |
11310
e0b7277e5394
[dataimport] PGHelper should be responsible to retrieve the database schema
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11095
diff
changeset
|
2 |
# copyright 2013-2016 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
3 |
# contact http://www.logilab.fr -- mailto:contact@logilab.fr |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
4 |
# |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
5 |
# This program is free software: you can redistribute it and/or modify it under |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
6 |
# the terms of the GNU Lesser General Public License as published by the Free |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
7 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
8 |
# any later version. |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
9 |
# |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
10 |
# This program is distributed in the hope that it will be useful, but WITHOUT |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
11 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
12 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
13 |
# details. |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
14 |
# |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
15 |
# You should have received a copy of the GNU Lesser General Public License along |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
16 |
# with this program. If not, see <http://www.gnu.org/licenses/>. |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
17 |
"""Massive store test case""" |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
18 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
19 |
import itertools |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
20 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
21 |
from cubicweb.devtools import testlib, PostgresApptestConfiguration |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
22 |
from cubicweb.devtools import startpgcluster, stoppgcluster |
11328
9f2d7da47526
[dataimport] test and fix external source support for the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11324
diff
changeset
|
23 |
from cubicweb.dataimport import ucsvreader, stores |
11360
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
24 |
from cubicweb.server.schema2sql import build_index_name |
11024
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
25 |
from cubicweb.dataimport.massive_store import MassiveObjectStore, PGHelper |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
26 |
|
11332
7187bf515251
[test] py.test related fix
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11329
diff
changeset
|
27 |
import test_stores |
11328
9f2d7da47526
[dataimport] test and fix external source support for the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11324
diff
changeset
|
28 |
|
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
29 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
30 |
def setUpModule(): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
31 |
startpgcluster(__file__) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
32 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
33 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
34 |
def tearDownModule(*args): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
35 |
stoppgcluster(__file__) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
36 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
37 |
|
11332
7187bf515251
[test] py.test related fix
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11329
diff
changeset
|
38 |
class MassiveObjectStoreWithCustomMDGenStoreTC( |
7187bf515251
[test] py.test related fix
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11329
diff
changeset
|
39 |
test_stores.NoHookRQLObjectStoreWithCustomMDGenStoreTC): |
11328
9f2d7da47526
[dataimport] test and fix external source support for the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11324
diff
changeset
|
40 |
configcls = PostgresApptestConfiguration |
9f2d7da47526
[dataimport] test and fix external source support for the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11324
diff
changeset
|
41 |
|
9f2d7da47526
[dataimport] test and fix external source support for the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11324
diff
changeset
|
42 |
def store_impl(self, cnx): |
9f2d7da47526
[dataimport] test and fix external source support for the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11324
diff
changeset
|
43 |
source = cnx.create_entity('CWSource', type=u'datafeed', name=u'test', url=u'test') |
9f2d7da47526
[dataimport] test and fix external source support for the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11324
diff
changeset
|
44 |
cnx.commit() |
9f2d7da47526
[dataimport] test and fix external source support for the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11324
diff
changeset
|
45 |
metagen = stores.MetadataGenerator(cnx, source=cnx.repo.sources_by_eid[source.eid]) |
9f2d7da47526
[dataimport] test and fix external source support for the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11324
diff
changeset
|
46 |
return MassiveObjectStore(cnx, metagen=metagen) |
9f2d7da47526
[dataimport] test and fix external source support for the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11324
diff
changeset
|
47 |
|
9f2d7da47526
[dataimport] test and fix external source support for the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11324
diff
changeset
|
48 |
|
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
49 |
class MassImportSimpleTC(testlib.CubicWebTC): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
50 |
configcls = PostgresApptestConfiguration |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
51 |
appid = 'data-massimport' |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
52 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
53 |
def cast(self, _type, value): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
54 |
try: |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
55 |
return _type(value) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
56 |
except ValueError: |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
57 |
return None |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
58 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
59 |
def push_geonames_data(self, dumpname, store): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
60 |
# Push timezones |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
61 |
cnx = store._cnx |
10938
a24a13742f3c
[test] Use datapath() in massive store tests
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
10907
diff
changeset
|
62 |
for code, gmt, dst, raw_offset in ucsvreader(open(self.datapath('timeZones.txt'), 'rb'), |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
63 |
delimiter='\t'): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
64 |
cnx.create_entity('TimeZone', code=code, gmt=float(gmt), |
11360
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
65 |
dst=float(dst), raw_offset=float(raw_offset)) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
66 |
timezone_code = dict(cnx.execute('Any C, X WHERE X is TimeZone, X code C')) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
67 |
# Push data |
10858
1a3e56e346d2
[dataimport/test] feed binary data to ucsvreader
Julien Cristau <julien.cristau@logilab.fr>
parents:
10857
diff
changeset
|
68 |
for ind, infos in enumerate(ucsvreader(open(dumpname, 'rb'), |
10868
ffb5b6c25cec
[dataimport/test] update call to ucsvreader
Julien Cristau <julien.cristau@logilab.fr>
parents:
10866
diff
changeset
|
69 |
delimiter='\t', |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
70 |
ignore_errors=True)): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
71 |
latitude = self.cast(float, infos[4]) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
72 |
longitude = self.cast(float, infos[5]) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
73 |
population = self.cast(int, infos[14]) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
74 |
elevation = self.cast(int, infos[15]) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
75 |
gtopo = self.cast(int, infos[16]) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
76 |
feature_class = infos[6] |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
77 |
if len(infos[6]) != 1: |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
78 |
feature_class = None |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
79 |
entity = {'name': infos[1], |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
80 |
'asciiname': infos[2], |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
81 |
'alternatenames': infos[3], |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
82 |
'latitude': latitude, 'longitude': longitude, |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
83 |
'feature_class': feature_class, |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
84 |
'alternate_country_code':infos[9], |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
85 |
'admin_code_3': infos[12], |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
86 |
'admin_code_4': infos[13], |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
87 |
'population': population, 'elevation': elevation, |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
88 |
'gtopo30': gtopo, 'timezone': timezone_code.get(infos[17]), |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
89 |
'cwuri': u'http://sws.geonames.org/%s/' % int(infos[0]), |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
90 |
'geonameid': int(infos[0]), |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
91 |
} |
10863
8e1f6de61300
[dataimport] implement new store API on massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10860
diff
changeset
|
92 |
store.prepare_insert_entity('Location', **entity) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
93 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
94 |
def test_autoflush_metadata(self): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
95 |
with self.admin_access.repo_cnx() as cnx: |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
96 |
crs = cnx.system_sql('SELECT * FROM entities WHERE type=%(t)s', |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
97 |
{'t': 'Location'}) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
98 |
self.assertEqual(len(crs.fetchall()), 0) |
10875
75d1b2d66f18
[dataimport] remove autoflush_metadata from MassiveObjectStore parameters
Julien Cristau <julien.cristau@logilab.fr>
parents:
10872
diff
changeset
|
99 |
store = MassiveObjectStore(cnx) |
10863
8e1f6de61300
[dataimport] implement new store API on massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10860
diff
changeset
|
100 |
store.prepare_insert_entity('Location', name=u'toto') |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
101 |
store.flush() |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
102 |
store.commit() |
10866
ed62ba97d79e
[dataimport/test] use store.finish instead of deprecated store.cleanup
Julien Cristau <julien.cristau@logilab.fr>
parents:
10863
diff
changeset
|
103 |
store.finish() |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
104 |
cnx.commit() |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
105 |
with self.admin_access.repo_cnx() as cnx: |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
106 |
crs = cnx.system_sql('SELECT * FROM entities WHERE type=%(t)s', |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
107 |
{'t': 'Location'}) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
108 |
self.assertEqual(len(crs.fetchall()), 1) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
109 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
110 |
def test_massimport_etype_metadata(self): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
111 |
with self.admin_access.repo_cnx() as cnx: |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
112 |
store = MassiveObjectStore(cnx) |
10863
8e1f6de61300
[dataimport] implement new store API on massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10860
diff
changeset
|
113 |
timezone_eid = store.prepare_insert_entity('TimeZone') |
8e1f6de61300
[dataimport] implement new store API on massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10860
diff
changeset
|
114 |
store.prepare_insert_entity('Location', timezone=timezone_eid) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
115 |
store.flush() |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
116 |
store.commit() |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
117 |
eid, etname = cnx.execute('Any X, TN WHERE X timezone TZ, X is T, ' |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
118 |
'T name TN')[0] |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
119 |
self.assertEqual(cnx.entity_from_eid(eid).cw_etype, etname) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
120 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
121 |
def test_drop_index(self): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
122 |
with self.admin_access.repo_cnx() as cnx: |
10869
575982c948a9
[dataimport] remove drop_index parameter from massive store
Julien Cristau <julien.cristau@logilab.fr>
parents:
10868
diff
changeset
|
123 |
store = MassiveObjectStore(cnx) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
124 |
cnx.commit() |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
125 |
with self.admin_access.repo_cnx() as cnx: |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
126 |
crs = cnx.system_sql('SELECT indexname FROM pg_indexes') |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
127 |
indexes = [r[0] for r in crs.fetchall()] |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
128 |
self.assertNotIn('entities_pkey', indexes) |
11095
02e88ca3bc23
[dataimport/test] fix test failures since 35b29f1eb37a
Julien Cristau <julien.cristau@logilab.fr>
parents:
11057
diff
changeset
|
129 |
self.assertNotIn('entities_extid_idx', indexes) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
130 |
self.assertNotIn('owned_by_relation_pkey', indexes) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
131 |
self.assertNotIn('owned_by_relation_to_idx', indexes) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
132 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
133 |
def test_drop_index_recreation(self): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
134 |
with self.admin_access.repo_cnx() as cnx: |
10869
575982c948a9
[dataimport] remove drop_index parameter from massive store
Julien Cristau <julien.cristau@logilab.fr>
parents:
10868
diff
changeset
|
135 |
store = MassiveObjectStore(cnx) |
10866
ed62ba97d79e
[dataimport/test] use store.finish instead of deprecated store.cleanup
Julien Cristau <julien.cristau@logilab.fr>
parents:
10863
diff
changeset
|
136 |
store.finish() |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
137 |
cnx.commit() |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
138 |
with self.admin_access.repo_cnx() as cnx: |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
139 |
crs = cnx.system_sql('SELECT indexname FROM pg_indexes') |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
140 |
indexes = [r[0] for r in crs.fetchall()] |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
141 |
self.assertIn('entities_pkey', indexes) |
11095
02e88ca3bc23
[dataimport/test] fix test failures since 35b29f1eb37a
Julien Cristau <julien.cristau@logilab.fr>
parents:
11057
diff
changeset
|
142 |
self.assertIn('entities_extid_idx', indexes) |
11360
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
143 |
self.assertIn(build_index_name('owned_by_relation', ['eid_from', 'eid_to'], 'key_'), |
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
144 |
indexes) |
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
145 |
self.assertIn(build_index_name('owned_by_relation', ['eid_from'], 'idx_'), |
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
146 |
indexes) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
147 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
148 |
def test_eids_seq_range(self): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
149 |
with self.admin_access.repo_cnx() as cnx: |
11028
66f94d7f9ca7
[dataimport] make eids_seq_range as massive store instance attribute again
Julien Cristau <julien.cristau@logilab.fr>
parents:
11026
diff
changeset
|
150 |
store = MassiveObjectStore(cnx, eids_seq_range=1000) |
11026
ce9b3886955d
[dataimport] remove eids_seq_start attribute from massive store
Julien Cristau <julien.cristau@logilab.fr>
parents:
11024
diff
changeset
|
151 |
store.restart_eid_sequence(50000) |
10863
8e1f6de61300
[dataimport] implement new store API on massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10860
diff
changeset
|
152 |
store.prepare_insert_entity('Location', name=u'toto') |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
153 |
store.flush() |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
154 |
cnx.commit() |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
155 |
with self.admin_access.repo_cnx() as cnx: |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
156 |
crs = cnx.system_sql("SELECT * FROM entities_id_seq") |
10860
252877c624f0
[dataimport/test] use the right assert methods instead of assertTrue with a comparison
Julien Cristau <julien.cristau@logilab.fr>
parents:
10858
diff
changeset
|
157 |
self.assertGreater(crs.fetchone()[0], 50000) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
158 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
159 |
def test_eid_entity(self): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
160 |
with self.admin_access.repo_cnx() as cnx: |
11028
66f94d7f9ca7
[dataimport] make eids_seq_range as massive store instance attribute again
Julien Cristau <julien.cristau@logilab.fr>
parents:
11026
diff
changeset
|
161 |
store = MassiveObjectStore(cnx, eids_seq_range=1000) |
11026
ce9b3886955d
[dataimport] remove eids_seq_start attribute from massive store
Julien Cristau <julien.cristau@logilab.fr>
parents:
11024
diff
changeset
|
162 |
store.restart_eid_sequence(50000) |
10863
8e1f6de61300
[dataimport] implement new store API on massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10860
diff
changeset
|
163 |
eid = store.prepare_insert_entity('Location', name=u'toto') |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
164 |
store.flush() |
10863
8e1f6de61300
[dataimport] implement new store API on massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10860
diff
changeset
|
165 |
self.assertGreater(eid, 50000) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
166 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
167 |
def test_eid_entity_2(self): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
168 |
with self.admin_access.repo_cnx() as cnx: |
11026
ce9b3886955d
[dataimport] remove eids_seq_start attribute from massive store
Julien Cristau <julien.cristau@logilab.fr>
parents:
11024
diff
changeset
|
169 |
store = MassiveObjectStore(cnx) |
ce9b3886955d
[dataimport] remove eids_seq_start attribute from massive store
Julien Cristau <julien.cristau@logilab.fr>
parents:
11024
diff
changeset
|
170 |
store.restart_eid_sequence(50000) |
10863
8e1f6de61300
[dataimport] implement new store API on massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10860
diff
changeset
|
171 |
eid = store.prepare_insert_entity('Location', name=u'toto', eid=10000) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
172 |
store.flush() |
10863
8e1f6de61300
[dataimport] implement new store API on massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10860
diff
changeset
|
173 |
self.assertEqual(eid, 10000) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
174 |
|
11024
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
175 |
@staticmethod |
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
176 |
def get_db_descr(cnx): |
11310
e0b7277e5394
[dataimport] PGHelper should be responsible to retrieve the database schema
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11095
diff
changeset
|
177 |
pgh = PGHelper(cnx) |
11024
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
178 |
all_tables = cnx.system_sql(''' |
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
179 |
SELECT table_name |
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
180 |
FROM information_schema.tables |
11310
e0b7277e5394
[dataimport] PGHelper should be responsible to retrieve the database schema
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11095
diff
changeset
|
181 |
where table_schema = %(s)s''', {'s': pgh.pg_schema}).fetchall() |
11024
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
182 |
all_tables_descr = {} |
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
183 |
for tablename, in all_tables: |
11314
c258bd6b20d8
[dataimport] rework PGHelper class
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11310
diff
changeset
|
184 |
all_tables_descr[tablename] = set(pgh.table_indexes(tablename)).union( |
c258bd6b20d8
[dataimport] rework PGHelper class
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11310
diff
changeset
|
185 |
set(pgh.table_constraints(tablename))) |
11024
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
186 |
return all_tables_descr |
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
187 |
|
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
188 |
def test_identical_schema(self): |
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
189 |
with self.admin_access.repo_cnx() as cnx: |
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
190 |
init_descr = self.get_db_descr(cnx) |
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
191 |
with self.admin_access.repo_cnx() as cnx: |
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
192 |
store = MassiveObjectStore(cnx) |
11321
fab543f542ac
[dataimport] inline some methods of the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11314
diff
changeset
|
193 |
store.prepare_insert_entity('Location', name=u'toto') |
11024
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
194 |
store.finish() |
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
195 |
with self.admin_access.repo_cnx() as cnx: |
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
196 |
final_descr = self.get_db_descr(cnx) |
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
197 |
self.assertEqual(init_descr, final_descr) |
dc70698dcf6c
[dataimport] check that MassiveObjectStore restores the db schema properly
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10938
diff
changeset
|
198 |
|
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
199 |
def test_on_commit_callback(self): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
200 |
counter = itertools.count() |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
201 |
with self.admin_access.repo_cnx() as cnx: |
11328
9f2d7da47526
[dataimport] test and fix external source support for the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11324
diff
changeset
|
202 |
store = MassiveObjectStore(cnx, on_commit_callback=lambda: next(counter)) |
10863
8e1f6de61300
[dataimport] implement new store API on massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10860
diff
changeset
|
203 |
store.prepare_insert_entity('Location', name=u'toto') |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
204 |
store.flush() |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
205 |
store.commit() |
11328
9f2d7da47526
[dataimport] test and fix external source support for the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11324
diff
changeset
|
206 |
self.assertEqual(next(counter), 1) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
207 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
208 |
def test_on_rollback_callback(self): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
209 |
counter = itertools.count() |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
210 |
with self.admin_access.repo_cnx() as cnx: |
10855
cd91f46fa633
[dataimport] use next builtin instead of next method on iterators
Julien Cristau <julien.cristau@logilab.fr>
parents:
10853
diff
changeset
|
211 |
store = MassiveObjectStore(cnx, on_rollback_callback=lambda *_: next(counter)) |
10863
8e1f6de61300
[dataimport] implement new store API on massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10860
diff
changeset
|
212 |
store.prepare_insert_entity('Location', nm='toto') |
11328
9f2d7da47526
[dataimport] test and fix external source support for the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11324
diff
changeset
|
213 |
store.commit() # commit modification to the database before flush |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
214 |
store.flush() |
11328
9f2d7da47526
[dataimport] test and fix external source support for the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11324
diff
changeset
|
215 |
self.assertEqual(next(counter), 1) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
216 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
217 |
def test_slave_mode_indexes(self): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
218 |
with self.admin_access.repo_cnx() as cnx: |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
219 |
slave_store = MassiveObjectStore(cnx, slave_mode=True) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
220 |
with self.admin_access.repo_cnx() as cnx: |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
221 |
crs = cnx.system_sql('SELECT indexname FROM pg_indexes') |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
222 |
indexes = [r[0] for r in crs.fetchall()] |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
223 |
self.assertIn('entities_pkey', indexes) |
11095
02e88ca3bc23
[dataimport/test] fix test failures since 35b29f1eb37a
Julien Cristau <julien.cristau@logilab.fr>
parents:
11057
diff
changeset
|
224 |
self.assertIn('entities_extid_idx', indexes) |
11360
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
225 |
self.assertIn(build_index_name('owned_by_relation', ['eid_from', 'eid_to'], 'key_'), |
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
226 |
indexes) |
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
227 |
self.assertIn(build_index_name('owned_by_relation', ['eid_from'], 'idx_'), |
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
228 |
indexes) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
229 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
230 |
def test_slave_mode_exception(self): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
231 |
with self.admin_access.repo_cnx() as cnx: |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
232 |
master_store = MassiveObjectStore(cnx, slave_mode=False) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
233 |
slave_store = MassiveObjectStore(cnx, slave_mode=True) |
10866
ed62ba97d79e
[dataimport/test] use store.finish instead of deprecated store.cleanup
Julien Cristau <julien.cristau@logilab.fr>
parents:
10863
diff
changeset
|
234 |
self.assertRaises(RuntimeError, slave_store.finish) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
235 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
236 |
def test_simple_insert(self): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
237 |
with self.admin_access.repo_cnx() as cnx: |
10875
75d1b2d66f18
[dataimport] remove autoflush_metadata from MassiveObjectStore parameters
Julien Cristau <julien.cristau@logilab.fr>
parents:
10872
diff
changeset
|
238 |
store = MassiveObjectStore(cnx) |
10938
a24a13742f3c
[test] Use datapath() in massive store tests
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
10907
diff
changeset
|
239 |
self.push_geonames_data(self.datapath('geonames.csv'), store) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
240 |
store.flush() |
10880
5fb592895e0f
[dataimport] remove implicit commits from MassiveObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
10875
diff
changeset
|
241 |
store.commit() |
5fb592895e0f
[dataimport] remove implicit commits from MassiveObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
10875
diff
changeset
|
242 |
store.finish() |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
243 |
with self.admin_access.repo_cnx() as cnx: |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
244 |
rset = cnx.execute('Any X WHERE X is Location') |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
245 |
self.assertEqual(len(rset), 4000) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
246 |
rset = cnx.execute('Any X WHERE X is Location, X timezone T') |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
247 |
self.assertEqual(len(rset), 4000) |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
248 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
249 |
def test_index_building(self): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
250 |
with self.admin_access.repo_cnx() as cnx: |
10875
75d1b2d66f18
[dataimport] remove autoflush_metadata from MassiveObjectStore parameters
Julien Cristau <julien.cristau@logilab.fr>
parents:
10872
diff
changeset
|
251 |
store = MassiveObjectStore(cnx) |
10938
a24a13742f3c
[test] Use datapath() in massive store tests
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
10907
diff
changeset
|
252 |
self.push_geonames_data(self.datapath('geonames.csv'), store) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
253 |
store.flush() |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
254 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
255 |
# Check index |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
256 |
crs = cnx.system_sql('SELECT indexname FROM pg_indexes') |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
257 |
indexes = [r[0] for r in crs.fetchall()] |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
258 |
self.assertNotIn('entities_pkey', indexes) |
11095
02e88ca3bc23
[dataimport/test] fix test failures since 35b29f1eb37a
Julien Cristau <julien.cristau@logilab.fr>
parents:
11057
diff
changeset
|
259 |
self.assertNotIn('entities_extid_idx', indexes) |
11360
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
260 |
self.assertNotIn(build_index_name('owned_by_relation', ['eid_from', 'eid_to'], 'key_'), |
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
261 |
indexes) |
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
262 |
self.assertNotIn(build_index_name('owned_by_relation', ['eid_from'], 'idx_'), |
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
263 |
indexes) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
264 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
265 |
# Cleanup -> index |
10866
ed62ba97d79e
[dataimport/test] use store.finish instead of deprecated store.cleanup
Julien Cristau <julien.cristau@logilab.fr>
parents:
10863
diff
changeset
|
266 |
store.finish() |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
267 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
268 |
# Check index again |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
269 |
crs = cnx.system_sql('SELECT indexname FROM pg_indexes') |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
270 |
indexes = [r[0] for r in crs.fetchall()] |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
271 |
self.assertIn('entities_pkey', indexes) |
11095
02e88ca3bc23
[dataimport/test] fix test failures since 35b29f1eb37a
Julien Cristau <julien.cristau@logilab.fr>
parents:
11057
diff
changeset
|
272 |
self.assertIn('entities_extid_idx', indexes) |
11360
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
273 |
self.assertIn(build_index_name('owned_by_relation', ['eid_from', 'eid_to'], 'key_'), |
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
274 |
indexes) |
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
275 |
self.assertIn(build_index_name('owned_by_relation', ['eid_from'], 'idx_'), |
49aca289134f
[sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11333
diff
changeset
|
276 |
indexes) |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
277 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
278 |
def test_multiple_insert(self): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
279 |
with self.admin_access.repo_cnx() as cnx: |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
280 |
store = MassiveObjectStore(cnx) |
11321
fab543f542ac
[dataimport] inline some methods of the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11314
diff
changeset
|
281 |
store.prepare_insert_entity('Location', name=u'toto') |
10866
ed62ba97d79e
[dataimport/test] use store.finish instead of deprecated store.cleanup
Julien Cristau <julien.cristau@logilab.fr>
parents:
10863
diff
changeset
|
282 |
store.finish() |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
283 |
store = MassiveObjectStore(cnx) |
11321
fab543f542ac
[dataimport] inline some methods of the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11314
diff
changeset
|
284 |
store.prepare_insert_entity('Location', name=u'toto') |
10866
ed62ba97d79e
[dataimport/test] use store.finish instead of deprecated store.cleanup
Julien Cristau <julien.cristau@logilab.fr>
parents:
10863
diff
changeset
|
285 |
store.finish() |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
286 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
287 |
def test_multiple_insert_relation(self): |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
288 |
with self.admin_access.repo_cnx() as cnx: |
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
289 |
store = MassiveObjectStore(cnx) |
11321
fab543f542ac
[dataimport] inline some methods of the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11314
diff
changeset
|
290 |
store.init_rtype_table('Country', 'used_language', 'Language') |
10866
ed62ba97d79e
[dataimport/test] use store.finish instead of deprecated store.cleanup
Julien Cristau <julien.cristau@logilab.fr>
parents:
10863
diff
changeset
|
291 |
store.finish() |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
292 |
store = MassiveObjectStore(cnx) |
11321
fab543f542ac
[dataimport] inline some methods of the massive store
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11314
diff
changeset
|
293 |
store.init_rtype_table('Country', 'used_language', 'Language') |
10866
ed62ba97d79e
[dataimport/test] use store.finish instead of deprecated store.cleanup
Julien Cristau <julien.cristau@logilab.fr>
parents:
10863
diff
changeset
|
294 |
store.finish() |
10853
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
295 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
296 |
|
de741492538d
[dataimport] backport massive store from dataio cube
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
297 |
if __name__ == '__main__': |
11333
094b29cf2dc9
[test] Prefer unittest.main
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11332
diff
changeset
|
298 |
import unittest |
094b29cf2dc9
[test] Prefer unittest.main
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11332
diff
changeset
|
299 |
unittest.main() |