equal
deleted
inserted
replaced
14 # |
14 # |
15 # You should have received a copy of the GNU Lesser General Public License along |
15 # You should have received a copy of the GNU Lesser General Public License along |
16 # with this program. If not, see <http://www.gnu.org/licenses/>. |
16 # with this program. If not, see <http://www.gnu.org/licenses/>. |
17 """Massive store test case""" |
17 """Massive store test case""" |
18 |
18 |
19 import os.path as osp |
|
20 import itertools |
19 import itertools |
21 |
20 |
22 from cubicweb.dataimport import ucsvreader |
21 from cubicweb.dataimport import ucsvreader |
23 from cubicweb.devtools import testlib, PostgresApptestConfiguration |
22 from cubicweb.devtools import testlib, PostgresApptestConfiguration |
24 from cubicweb.devtools import startpgcluster, stoppgcluster |
23 from cubicweb.devtools import startpgcluster, stoppgcluster |
25 from cubicweb.dataimport.massive_store import MassiveObjectStore |
24 from cubicweb.dataimport.massive_store import MassiveObjectStore |
26 |
|
27 |
|
28 HERE = osp.abspath(osp.dirname(__file__)) |
|
29 |
25 |
30 |
26 |
31 def setUpModule(): |
27 def setUpModule(): |
32 startpgcluster(__file__) |
28 startpgcluster(__file__) |
33 |
29 |
47 return None |
43 return None |
48 |
44 |
49 def push_geonames_data(self, dumpname, store): |
45 def push_geonames_data(self, dumpname, store): |
50 # Push timezones |
46 # Push timezones |
51 cnx = store._cnx |
47 cnx = store._cnx |
52 for code, gmt, dst, raw_offset in ucsvreader(open(osp.join(HERE, 'data/timeZones.txt'), 'rb'), |
48 for code, gmt, dst, raw_offset in ucsvreader(open(self.datapath('timeZones.txt'), 'rb'), |
53 delimiter='\t'): |
49 delimiter='\t'): |
54 cnx.create_entity('TimeZone', code=code, gmt=float(gmt), |
50 cnx.create_entity('TimeZone', code=code, gmt=float(gmt), |
55 dst=float(dst), raw_offset=float(raw_offset)) |
51 dst=float(dst), raw_offset=float(raw_offset)) |
56 timezone_code = dict(cnx.execute('Any C, X WHERE X is TimeZone, X code C')) |
52 timezone_code = dict(cnx.execute('Any C, X WHERE X is TimeZone, X code C')) |
57 # Push data |
53 # Push data |
206 self.assertRaises(RuntimeError, slave_store.finish) |
202 self.assertRaises(RuntimeError, slave_store.finish) |
207 |
203 |
208 def test_simple_insert(self): |
204 def test_simple_insert(self): |
209 with self.admin_access.repo_cnx() as cnx: |
205 with self.admin_access.repo_cnx() as cnx: |
210 store = MassiveObjectStore(cnx) |
206 store = MassiveObjectStore(cnx) |
211 self.push_geonames_data(osp.join(HERE, 'data/geonames.csv'), store) |
207 self.push_geonames_data(self.datapath('geonames.csv'), store) |
212 store.flush() |
208 store.flush() |
213 store.commit() |
209 store.commit() |
214 store.finish() |
210 store.finish() |
215 with self.admin_access.repo_cnx() as cnx: |
211 with self.admin_access.repo_cnx() as cnx: |
216 rset = cnx.execute('Any X WHERE X is Location') |
212 rset = cnx.execute('Any X WHERE X is Location') |
219 self.assertEqual(len(rset), 4000) |
215 self.assertEqual(len(rset), 4000) |
220 |
216 |
221 def test_index_building(self): |
217 def test_index_building(self): |
222 with self.admin_access.repo_cnx() as cnx: |
218 with self.admin_access.repo_cnx() as cnx: |
223 store = MassiveObjectStore(cnx) |
219 store = MassiveObjectStore(cnx) |
224 self.push_geonames_data(osp.join(HERE, 'data/geonames.csv'), store) |
220 self.push_geonames_data(self.datapath('geonames.csv'), store) |
225 store.flush() |
221 store.flush() |
226 |
222 |
227 # Check index |
223 # Check index |
228 crs = cnx.system_sql('SELECT indexname FROM pg_indexes') |
224 crs = cnx.system_sql('SELECT indexname FROM pg_indexes') |
229 indexes = [r[0] for r in crs.fetchall()] |
225 indexes = [r[0] for r in crs.fetchall()] |