author | Alexandre Fayolle <alexandre.fayolle@logilab.fr> |
Wed, 02 Jun 2010 18:12:27 +0200 | |
branch | stable |
changeset 5644 | 73d8a757db80 |
parent 5424 | 8ecbcbff9777 |
child 5557 | 1a534c596bff |
permissions | -rw-r--r-- |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
1 |
# -*- coding: utf-8 -*- |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5097
diff
changeset
|
2 |
# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5097
diff
changeset
|
3 |
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5097
diff
changeset
|
4 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5097
diff
changeset
|
5 |
# This file is part of CubicWeb. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5097
diff
changeset
|
6 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5097
diff
changeset
|
7 |
# CubicWeb is free software: you can redistribute it and/or modify it under the |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5097
diff
changeset
|
8 |
# terms of the GNU Lesser General Public License as published by the Free |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5097
diff
changeset
|
9 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5097
diff
changeset
|
10 |
# any later version. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5097
diff
changeset
|
11 |
# |
5424
8ecbcbff9777
replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5421
diff
changeset
|
12 |
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5097
diff
changeset
|
13 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5097
diff
changeset
|
14 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5097
diff
changeset
|
15 |
# details. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5097
diff
changeset
|
16 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5097
diff
changeset
|
17 |
# You should have received a copy of the GNU Lesser General Public License along |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5097
diff
changeset
|
18 |
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
19 |
"""This module provides tools to import tabular data. |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
20 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
21 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
22 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
23 |
Example of use (run this with `cubicweb-ctl shell instance import-script.py`): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
24 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
25 |
.. sourcecode:: python |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
26 |
|
3318
5b47b9f09bca
documentation : fixed docstring
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
3029
diff
changeset
|
27 |
from cubicweb.devtools.dataimport import * |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
28 |
# define data generators |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
29 |
GENERATORS = [] |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
30 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
31 |
USERS = [('Prenom', 'firstname', ()), |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
32 |
('Nom', 'surname', ()), |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
33 |
('Identifiant', 'login', ()), |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
34 |
] |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
35 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
36 |
def gen_users(ctl): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
37 |
for row in ctl.get_data('utilisateurs'): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
38 |
entity = mk_entity(row, USERS) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
39 |
entity['upassword'] = u'motdepasse' |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
40 |
ctl.check('login', entity['login'], None) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
41 |
ctl.store.add('CWUser', entity) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
42 |
email = {'address': row['email']} |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
43 |
ctl.store.add('EmailAddress', email) |
3003
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
44 |
ctl.store.relate(entity['eid'], 'use_email', email['eid']) |
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
45 |
ctl.store.rql('SET U in_group G WHERE G name "users", U eid %(x)s', {'x':entity['eid']}) |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
46 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
47 |
CHK = [('login', check_doubles, 'Utilisateurs Login', |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
48 |
'Deux utilisateurs ne devraient pas avoir le même login.'), |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
49 |
] |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
50 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
51 |
GENERATORS.append( (gen_users, CHK) ) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
52 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
53 |
# create controller |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
54 |
ctl = CWImportController(RQLObjectStore(cnx)) |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
55 |
ctl.askerror = 1 |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
56 |
ctl.generators = GENERATORS |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
57 |
ctl.data['utilisateurs'] = lazytable(utf8csvreader(open('users.csv'))) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
58 |
# run |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
59 |
ctl.run() |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
60 |
|
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
61 |
.. BUG file with one column are not parsable |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
62 |
.. TODO rollback() invocation is not possible yet |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
63 |
""" |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
64 |
__docformat__ = "restructuredtext en" |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
65 |
|
4186
ca7e526b07b6
import cleanup, check data file exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4173
diff
changeset
|
66 |
import sys |
ca7e526b07b6
import cleanup, check data file exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4173
diff
changeset
|
67 |
import csv |
ca7e526b07b6
import cleanup, check data file exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4173
diff
changeset
|
68 |
import traceback |
ca7e526b07b6
import cleanup, check data file exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4173
diff
changeset
|
69 |
import os.path as osp |
ca7e526b07b6
import cleanup, check data file exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4173
diff
changeset
|
70 |
from StringIO import StringIO |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
71 |
from copy import copy |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
72 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
73 |
from logilab.common import shellutils |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
74 |
from logilab.common.date import strptime |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
75 |
from logilab.common.decorators import cached |
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
76 |
from logilab.common.deprecation import deprecated |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
77 |
|
5066
bf5cbc351e99
[repo] move eschema_eid function from hooks.metadata to server.utils
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5063
diff
changeset
|
78 |
from cubicweb.server.utils import eschema_eid |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
79 |
|
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
80 |
def ucsvreader_pb(filepath, encoding='utf-8', separator=',', quote='"', |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
81 |
skipfirst=False, withpb=True): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
82 |
"""same as ucsvreader but a progress bar is displayed as we iter on rows""" |
4186
ca7e526b07b6
import cleanup, check data file exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4173
diff
changeset
|
83 |
if not osp.exists(filepath): |
ca7e526b07b6
import cleanup, check data file exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4173
diff
changeset
|
84 |
raise Exception("file doesn't exists: %s" % filepath) |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
85 |
rowcount = int(shellutils.Execute('wc -l "%s"' % filepath).out.strip().split()[0]) |
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
86 |
if skipfirst: |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
87 |
rowcount -= 1 |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
88 |
if withpb: |
4140
46ddd27a4ca4
tweaks output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4136
diff
changeset
|
89 |
pb = shellutils.ProgressBar(rowcount, 50) |
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
90 |
for urow in ucsvreader(file(filepath), encoding, separator, quote, skipfirst): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
91 |
yield urow |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
92 |
if withpb: |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
93 |
pb.update() |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
94 |
print ' %s rows imported' % rowcount |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
95 |
|
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
96 |
def ucsvreader(stream, encoding='utf-8', separator=',', quote='"', |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
97 |
skipfirst=False): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
98 |
"""A csv reader that accepts files with any encoding and outputs unicode |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
99 |
strings |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
100 |
""" |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
101 |
it = iter(csv.reader(stream, delimiter=separator, quotechar=quote)) |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
102 |
if skipfirst: |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
103 |
it.next() |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
104 |
for row in it: |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
105 |
yield [item.decode(encoding) for item in row] |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
106 |
|
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
107 |
def commit_every(nbit, store, it): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
108 |
for i, x in enumerate(it): |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
109 |
yield x |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
110 |
if nbit is not None and i % nbit: |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
111 |
store.commit() |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
112 |
if nbit is not None: |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
113 |
store.commit() |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
114 |
|
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
115 |
def lazytable(reader): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
116 |
"""The first row is taken to be the header of the table and |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
117 |
used to output a dict for each row of data. |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
118 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
119 |
>>> data = lazytable(utf8csvreader(open(filename))) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
120 |
""" |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
121 |
header = reader.next() |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
122 |
for row in reader: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
123 |
yield dict(zip(header, row)) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
124 |
|
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
125 |
def mk_entity(row, map): |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
126 |
"""Return a dict made from sanitized mapped values. |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
127 |
|
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
128 |
ValueError can be raised on unexpected values found in checkers |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
129 |
|
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
130 |
>>> row = {'myname': u'dupont'} |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
131 |
>>> map = [('myname', u'name', (call_transform_method('title'),))] |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
132 |
>>> mk_entity(row, map) |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
133 |
{'name': u'Dupont'} |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
134 |
>>> row = {'myname': u'dupont', 'optname': u''} |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
135 |
>>> map = [('myname', u'name', (call_transform_method('title'),)), |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
136 |
... ('optname', u'MARKER', (optional,))] |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
137 |
>>> mk_entity(row, map) |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
138 |
{'name': u'Dupont', 'optname': None} |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
139 |
""" |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
140 |
res = {} |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
141 |
assert isinstance(row, dict) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
142 |
assert isinstance(map, list) |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
143 |
for src, dest, funcs in map: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
144 |
res[dest] = row[src] |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
145 |
try: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
146 |
for func in funcs: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
147 |
res[dest] = func(res[dest]) |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
148 |
if res[dest] is None: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
149 |
break |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
150 |
except ValueError, err: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
151 |
raise ValueError('error with %r field: %s' % (src, err)) |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
152 |
return res |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
153 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
154 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
155 |
# user interactions ############################################################ |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
156 |
|
3029
bc573d5fb5b7
F [devtools] by default dataimport prints message on stdout
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
3003
diff
changeset
|
157 |
def tell(msg): |
bc573d5fb5b7
F [devtools] by default dataimport prints message on stdout
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
3003
diff
changeset
|
158 |
print msg |
bc573d5fb5b7
F [devtools] by default dataimport prints message on stdout
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
3003
diff
changeset
|
159 |
|
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
160 |
def confirm(question): |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
161 |
"""A confirm function that asks for yes/no/abort and exits on abort.""" |
4721
8f63691ccb7f
pylint style fixes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4613
diff
changeset
|
162 |
answer = shellutils.ASK.ask(question, ('Y', 'n', 'abort'), 'Y') |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
163 |
if answer == 'abort': |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
164 |
sys.exit(1) |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
165 |
return answer == 'Y' |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
166 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
167 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
168 |
class catch_error(object): |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
169 |
"""Helper for @contextmanager decorator.""" |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
170 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
171 |
def __init__(self, ctl, key='unexpected error', msg=None): |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
172 |
self.ctl = ctl |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
173 |
self.key = key |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
174 |
self.msg = msg |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
175 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
176 |
def __enter__(self): |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
177 |
return self |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
178 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
179 |
def __exit__(self, type, value, traceback): |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
180 |
if type is not None: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
181 |
if issubclass(type, (KeyboardInterrupt, SystemExit)): |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
182 |
return # re-raise |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
183 |
if self.ctl.catcherrors: |
4173
cfd5d3270f99
msg isn't defined there, but we've to give traceback information to record error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4152
diff
changeset
|
184 |
self.ctl.record_error(self.key, None, type, value, traceback) |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
185 |
return True # silent |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
186 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
187 |
|
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
188 |
# base sanitizing/coercing functions ########################################### |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
189 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
190 |
def optional(value): |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
191 |
"""checker to filter optional field |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
192 |
|
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
193 |
If value is undefined (ex: empty string), return None that will |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
194 |
break the checkers validation chain |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
195 |
|
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
196 |
General use is to add 'optional' check in first condition to avoid |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
197 |
ValueError by further checkers |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
198 |
|
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
199 |
>>> MAPPER = [(u'value', 'value', (optional, int))] |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
200 |
>>> row = {'value': u'XXX'} |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
201 |
>>> mk_entity(row, MAPPER) |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
202 |
{'value': None} |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
203 |
>>> row = {'value': u'100'} |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
204 |
>>> mk_entity(row, MAPPER) |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
205 |
{'value': 100} |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
206 |
""" |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
207 |
if value: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
208 |
return value |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
209 |
return None |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
210 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
211 |
def required(value): |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
212 |
"""raise ValueError is value is empty |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
213 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
214 |
This check should be often found in last position in the chain. |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
215 |
""" |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
216 |
if value: |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
217 |
return value |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
218 |
raise ValueError("required") |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
219 |
|
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
220 |
def todatetime(format='%d/%m/%Y'): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
221 |
"""return a transformation function to turn string input value into a |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
222 |
`datetime.datetime` instance, using given format. |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
223 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
224 |
Follow it by `todate` or `totime` functions from `logilab.common.date` if |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
225 |
you want a `date`/`time` instance instead of `datetime`. |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
226 |
""" |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
227 |
def coerce(value): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
228 |
return strptime(value, format) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
229 |
return coerce |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
230 |
|
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
231 |
def call_transform_method(methodname, *args, **kwargs): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
232 |
"""return value returned by calling the given method on input""" |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
233 |
def coerce(value): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
234 |
return getattr(value, methodname)(*args, **kwargs) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
235 |
return coerce |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
236 |
|
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
237 |
def call_check_method(methodname, *args, **kwargs): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
238 |
"""check value returned by calling the given method on input is true, |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
239 |
else raise ValueError |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
240 |
""" |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
241 |
def check(value): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
242 |
if getattr(value, methodname)(*args, **kwargs): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
243 |
return value |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
244 |
raise ValueError('%s not verified on %r' % (methodname, value)) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
245 |
return check |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
246 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
247 |
# base integrity checking functions ############################################ |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
248 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
249 |
def check_doubles(buckets): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
250 |
"""Extract the keys that have more than one item in their bucket.""" |
4721
8f63691ccb7f
pylint style fixes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4613
diff
changeset
|
251 |
return [(k, len(v)) for k, v in buckets.items() if len(v) > 1] |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
252 |
|
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
253 |
def check_doubles_not_none(buckets): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
254 |
"""Extract the keys that have more than one item in their bucket.""" |
4721
8f63691ccb7f
pylint style fixes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4613
diff
changeset
|
255 |
return [(k, len(v)) for k, v in buckets.items() |
8f63691ccb7f
pylint style fixes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4613
diff
changeset
|
256 |
if k is not None and len(v) > 1] |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
257 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
258 |
|
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
259 |
# object stores ################################################################# |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
260 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
261 |
class ObjectStore(object): |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
262 |
"""Store objects in memory for *faster* validation (development mode) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
263 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
264 |
But it will not enforce the constraints of the schema and hence will miss some problems |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
265 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
266 |
>>> store = ObjectStore() |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
267 |
>>> user = {'login': 'johndoe'} |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
268 |
>>> store.add('CWUser', user) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
269 |
>>> group = {'name': 'unknown'} |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
270 |
>>> store.add('CWUser', group) |
3003
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
271 |
>>> store.relate(user['eid'], 'in_group', group['eid']) |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
272 |
""" |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
273 |
def __init__(self): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
274 |
self.items = [] |
3003
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
275 |
self.eids = {} |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
276 |
self.types = {} |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
277 |
self.relations = set() |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
278 |
self.indexes = {} |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
279 |
self._rql = None |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
280 |
self._commit = None |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
281 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
282 |
def _put(self, type, item): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
283 |
self.items.append(item) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
284 |
return len(self.items) - 1 |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
285 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
286 |
def add(self, type, item): |
3486
ea6bf6f9ba0c
[cwctl] improve dialog messages
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
3318
diff
changeset
|
287 |
assert isinstance(item, dict), 'item is not a dict but a %s' % type(item) |
3003
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
288 |
eid = item['eid'] = self._put(type, item) |
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
289 |
self.eids[eid] = item |
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
290 |
self.types.setdefault(type, []).append(eid) |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
291 |
|
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
292 |
def relate(self, eid_from, rtype, eid_to, inlined=False): |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
293 |
"""Add new relation""" |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
294 |
relation = eid_from, rtype, eid_to |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
295 |
self.relations.add(relation) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
296 |
return relation |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
297 |
|
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
298 |
def commit(self): |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
299 |
"""this commit method do nothing by default |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
300 |
|
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
301 |
This is voluntary to use the frequent autocommit feature in CubicWeb |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
302 |
when you are using hooks or another |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
303 |
|
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
304 |
If you want override commit method, please set it by the |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
305 |
constructor |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
306 |
""" |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
307 |
pass |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
308 |
|
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
309 |
def rql(self, *args): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
310 |
if self._rql is not None: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
311 |
return self._rql(*args) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
312 |
|
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
313 |
@property |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
314 |
def nb_inserted_entities(self): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
315 |
return len(self.eids) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
316 |
@property |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
317 |
def nb_inserted_types(self): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
318 |
return len(self.types) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
319 |
@property |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
320 |
def nb_inserted_relations(self): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
321 |
return len(self.relations) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
322 |
|
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
323 |
@deprecated("[3.7] index support will disappear") |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
324 |
def build_index(self, name, type, func=None, can_be_empty=False): |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
325 |
"""build internal index for further search""" |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
326 |
index = {} |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
327 |
if func is None or not callable(func): |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
328 |
func = lambda x: x['eid'] |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
329 |
for eid in self.types[type]: |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
330 |
index.setdefault(func(self.eids[eid]), []).append(eid) |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
331 |
if not can_be_empty: |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
332 |
assert index, "new index '%s' cannot be empty" % name |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
333 |
self.indexes[name] = index |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
334 |
|
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
335 |
@deprecated("[3.7] index support will disappear") |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
336 |
def build_rqlindex(self, name, type, key, rql, rql_params=False, |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
337 |
func=None, can_be_empty=False): |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
338 |
"""build an index by rql query |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
339 |
|
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
340 |
rql should return eid in first column |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
341 |
ctl.store.build_index('index_name', 'users', 'login', 'Any U WHERE U is CWUser') |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
342 |
""" |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
343 |
self.types[type] = [] |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
344 |
rset = self.rql(rql, rql_params or {}) |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
345 |
if not can_be_empty: |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
346 |
assert rset, "new index type '%s' cannot be empty (0 record found)" % type |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
347 |
for entity in rset.entities(): |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
348 |
getattr(entity, key) # autopopulate entity with key attribute |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
349 |
self.eids[entity.eid] = dict(entity) |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
350 |
if entity.eid not in self.types[type]: |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
351 |
self.types[type].append(entity.eid) |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
352 |
|
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
353 |
# Build index with specified key |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
354 |
func = lambda x: x[key] |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
355 |
self.build_index(name, type, func, can_be_empty=can_be_empty) |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
356 |
|
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
357 |
@deprecated("[3.7] index support will disappear") |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
358 |
def fetch(self, name, key, unique=False, decorator=None): |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
359 |
"""index fetcher method |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
360 |
|
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
361 |
decorator is a callable method or an iterator of callable methods (usually a lambda function) |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
362 |
decorator=lambda x: x[:1] (first value is returned) |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
363 |
decorator=lambda x: x.lower (lowercased value is returned) |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
364 |
|
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
365 |
decorator is handy when you want to improve index keys but without |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
366 |
changing the original field |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
367 |
|
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
368 |
Same check functions can be reused here. |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
369 |
""" |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
370 |
eids = self.indexes[name].get(key, []) |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
371 |
if decorator is not None: |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
372 |
if not hasattr(decorator, '__iter__'): |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
373 |
decorator = (decorator,) |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
374 |
for f in decorator: |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
375 |
eids = f(eids) |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
376 |
if unique: |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
377 |
assert len(eids) == 1, u'expected a single one value for key "%s" in index "%s". Got %i' % (key, name, len(eids)) |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
378 |
eids = eids[0] |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
379 |
return eids |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
380 |
|
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
381 |
@deprecated("[3.7] index support will disappear") |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
382 |
def find(self, type, key, value): |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
383 |
for idx in self.types[type]: |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
384 |
item = self.items[idx] |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
385 |
if item[key] == value: |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
386 |
yield item |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
387 |
|
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
388 |
@deprecated("[3.7] checkpoint() deprecated. use commit() instead") |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
389 |
def checkpoint(self): |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
390 |
self.commit() |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
391 |
|
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
392 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
393 |
class RQLObjectStore(ObjectStore): |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
394 |
"""ObjectStore that works with an actual RQL repository (production mode)""" |
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
395 |
_rql = None # bw compat |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
396 |
|
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
397 |
def __init__(self, session=None, commit=None): |
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
398 |
ObjectStore.__init__(self) |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
399 |
if session is not None: |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
400 |
if not hasattr(session, 'set_pool'): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
401 |
# connection |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
402 |
cnx = session |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
403 |
session = session.request() |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
404 |
session.set_pool = lambda : None |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
405 |
commit = commit or cnx.commit |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
406 |
else: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
407 |
session.set_pool() |
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
408 |
self.session = session |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
409 |
self._commit = commit or session.commit |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
410 |
elif commit is not None: |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
411 |
self._commit = commit |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
412 |
# XXX .session |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
413 |
|
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
414 |
@deprecated("[3.7] checkpoint() deprecated. use commit() instead") |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
415 |
def checkpoint(self): |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
416 |
self.commit() |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
417 |
|
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
418 |
def commit(self): |
5063
2a94b61837e1
[dataimport] stop disabling undo ; commit return transaction id
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5054
diff
changeset
|
419 |
txuuid = self._commit() |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
420 |
self.session.set_pool() |
5063
2a94b61837e1
[dataimport] stop disabling undo ; commit return transaction id
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5054
diff
changeset
|
421 |
return txuuid |
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
422 |
|
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
423 |
def rql(self, *args): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
424 |
if self._rql is not None: |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
425 |
return self._rql(*args) |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
426 |
return self.session.execute(*args) |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
427 |
|
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
428 |
def create_entity(self, *args, **kwargs): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
429 |
entity = self.session.create_entity(*args, **kwargs) |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
430 |
self.eids[entity.eid] = entity |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
431 |
self.types.setdefault(args[0], []).append(entity.eid) |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
432 |
return entity |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
433 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
434 |
def _put(self, type, item): |
4721
8f63691ccb7f
pylint style fixes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4613
diff
changeset
|
435 |
query = ('INSERT %s X: ' % type) + ', '.join('X %s %%(%s)s' % (k, k) |
4734 | 436 |
for k in item) |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
437 |
return self.rql(query, item)[0][0] |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
438 |
|
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
439 |
def relate(self, eid_from, rtype, eid_to, inlined=False): |
4721
8f63691ccb7f
pylint style fixes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4613
diff
changeset
|
440 |
eid_from, rtype, eid_to = super(RQLObjectStore, self).relate( |
8f63691ccb7f
pylint style fixes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4613
diff
changeset
|
441 |
eid_from, rtype, eid_to) |
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
442 |
self.rql('SET X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype, |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
443 |
{'x': int(eid_from), 'y': int(eid_to)}, ('x', 'y')) |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
444 |
|
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
445 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
446 |
# the import controller ######################################################## |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
447 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
448 |
class CWImportController(object): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
449 |
"""Controller of the data import process. |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
450 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
451 |
>>> ctl = CWImportController(store) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
452 |
>>> ctl.generators = list_of_data_generators |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
453 |
>>> ctl.data = dict_of_data_tables |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
454 |
>>> ctl.run() |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
455 |
""" |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
456 |
|
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
457 |
def __init__(self, store, askerror=0, catcherrors=None, tell=tell, |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
458 |
commitevery=50): |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
459 |
self.store = store |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
460 |
self.generators = None |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
461 |
self.data = {} |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
462 |
self.errors = None |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
463 |
self.askerror = askerror |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
464 |
if catcherrors is None: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
465 |
catcherrors = askerror |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
466 |
self.catcherrors = catcherrors |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
467 |
self.commitevery = commitevery # set to None to do a single commit |
3029
bc573d5fb5b7
F [devtools] by default dataimport prints message on stdout
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
3003
diff
changeset
|
468 |
self._tell = tell |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
469 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
470 |
def check(self, type, key, value): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
471 |
self._checks.setdefault(type, {}).setdefault(key, []).append(value) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
472 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
473 |
def check_map(self, entity, key, map, default): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
474 |
try: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
475 |
entity[key] = map[entity[key]] |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
476 |
except KeyError: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
477 |
self.check(key, entity[key], None) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
478 |
entity[key] = default |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
479 |
|
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
480 |
def record_error(self, key, msg=None, type=None, value=None, tb=None): |
4186
ca7e526b07b6
import cleanup, check data file exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4173
diff
changeset
|
481 |
tmp = StringIO() |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
482 |
if type is None: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
483 |
traceback.print_exc(file=tmp) |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
484 |
else: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
485 |
traceback.print_exception(type, value, tb, file=tmp) |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
486 |
print tmp.getvalue() |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
487 |
# use a list to avoid counting a <nb lines> errors instead of one |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
488 |
errorlog = self.errors.setdefault(key, []) |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
489 |
if msg is None: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
490 |
errorlog.append(tmp.getvalue().splitlines()) |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
491 |
else: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
492 |
errorlog.append( (msg, tmp.getvalue().splitlines()) ) |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
493 |
|
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
494 |
def run(self): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
495 |
self.errors = {} |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
496 |
for func, checks in self.generators: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
497 |
self._checks = {} |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
498 |
func_name = func.__name__ |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
499 |
self.tell("Run import function '%s'..." % func_name) |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
500 |
try: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
501 |
func(self) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
502 |
except: |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
503 |
if self.catcherrors: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
504 |
self.record_error(func_name, 'While calling %s' % func.__name__) |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
505 |
else: |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
506 |
self._print_stats() |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
507 |
raise |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
508 |
for key, func, title, help in checks: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
509 |
buckets = self._checks.get(key) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
510 |
if buckets: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
511 |
err = func(buckets) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
512 |
if err: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
513 |
self.errors[title] = (help, err) |
5097
60a237638f57
[dataimport] print transaction id when we get one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5066
diff
changeset
|
514 |
txuuid = self.store.commit() |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
515 |
self._print_stats() |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
516 |
if self.errors: |
4721
8f63691ccb7f
pylint style fixes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4613
diff
changeset
|
517 |
if self.askerror == 2 or (self.askerror and confirm('Display errors ?')): |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
518 |
from pprint import pformat |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
519 |
for errkey, error in self.errors.items(): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
520 |
self.tell("\n%s (%s): %d\n" % (error[0], errkey, len(error[1]))) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
521 |
self.tell(pformat(sorted(error[1]))) |
5097
60a237638f57
[dataimport] print transaction id when we get one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5066
diff
changeset
|
522 |
if txuuid is not None: |
60a237638f57
[dataimport] print transaction id when we get one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5066
diff
changeset
|
523 |
print 'transaction id:', txuuid |
4912
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
524 |
def _print_stats(self): |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
525 |
nberrors = sum(len(err[1]) for err in self.errors.values()) |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
526 |
self.tell('\nImport statistics: %i entities, %i types, %i relations and %i errors' |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
527 |
% (self.store.nb_inserted_entities, |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
528 |
self.store.nb_inserted_types, |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
529 |
self.store.nb_inserted_relations, |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
530 |
nberrors)) |
9767cc516b4f
[R] dataimport: changes
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4847
diff
changeset
|
531 |
|
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
532 |
def get_data(self, key): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
533 |
return self.data.get(key) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
534 |
|
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
535 |
def index(self, name, key, value, unique=False): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
536 |
"""create a new index |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
537 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
538 |
If unique is set to True, only first occurence will be kept not the following ones |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
539 |
""" |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
540 |
if unique: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
541 |
try: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
542 |
if value in self.store.indexes[name][key]: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
543 |
return |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
544 |
except KeyError: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
545 |
# we're sure that one is the first occurence; so continue... |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
546 |
pass |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
547 |
self.store.indexes.setdefault(name, {}).setdefault(key, []).append(value) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
548 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
549 |
def tell(self, msg): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
550 |
self._tell(msg) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
551 |
|
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
552 |
def iter_and_commit(self, datakey): |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
553 |
"""iter rows, triggering commit every self.commitevery iterations""" |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
554 |
return commit_every(self.commitevery, self.store, self.get_data(datakey)) |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
555 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
556 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
557 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
558 |
from datetime import datetime |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
559 |
from cubicweb.schema import META_RTYPES, VIRTUAL_RTYPES |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
560 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
561 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
562 |
class NoHookRQLObjectStore(RQLObjectStore): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
563 |
"""ObjectStore that works with an actual RQL repository (production mode)""" |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
564 |
_rql = None # bw compat |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
565 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
566 |
def __init__(self, session, metagen=None, baseurl=None): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
567 |
super(NoHookRQLObjectStore, self).__init__(session) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
568 |
self.source = session.repo.system_source |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
569 |
self.rschema = session.repo.schema.rschema |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
570 |
self.add_relation = self.source.add_relation |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
571 |
if metagen is None: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
572 |
metagen = MetaGenerator(session, baseurl) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
573 |
self.metagen = metagen |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
574 |
self._nb_inserted_entities = 0 |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
575 |
self._nb_inserted_types = 0 |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
576 |
self._nb_inserted_relations = 0 |
5054
cb066d29166a
fix dataimport for 3.7.2
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
577 |
self.rql = session.execute |
cb066d29166a
fix dataimport for 3.7.2
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
578 |
# deactivate security |
cb066d29166a
fix dataimport for 3.7.2
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
579 |
session.set_read_security(False) |
cb066d29166a
fix dataimport for 3.7.2
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
580 |
session.set_write_security(False) |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
581 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
582 |
def create_entity(self, etype, **kwargs): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
583 |
for k, v in kwargs.iteritems(): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
584 |
kwargs[k] = getattr(v, 'eid', v) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
585 |
entity, rels = self.metagen.base_etype_dicts(etype) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
586 |
entity = copy(entity) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
587 |
entity._related_cache = {} |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
588 |
self.metagen.init_entity(entity) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
589 |
entity.update(kwargs) |
5054
cb066d29166a
fix dataimport for 3.7.2
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
590 |
entity.edited_attributes = set(entity) |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
591 |
session = self.session |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
592 |
self.source.add_entity(session, entity) |
5054
cb066d29166a
fix dataimport for 3.7.2
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
593 |
self.source.add_info(session, entity, self.source, None, complete=False) |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
594 |
for rtype, targeteids in rels.iteritems(): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
595 |
# targeteids may be a single eid or a list of eids |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
596 |
inlined = self.rschema(rtype).inlined |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
597 |
try: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
598 |
for targeteid in targeteids: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
599 |
self.add_relation(session, entity.eid, rtype, targeteid, |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
600 |
inlined) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
601 |
except TypeError: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
602 |
self.add_relation(session, entity.eid, rtype, targeteids, |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
603 |
inlined) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
604 |
self._nb_inserted_entities += 1 |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
605 |
return entity |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
606 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
607 |
def relate(self, eid_from, rtype, eid_to): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
608 |
assert not rtype.startswith('reverse_') |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
609 |
self.add_relation(self.session, eid_from, rtype, eid_to, |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
610 |
self.rschema(rtype).inlined) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
611 |
self._nb_inserted_relations += 1 |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
612 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
613 |
@property |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
614 |
def nb_inserted_entities(self): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
615 |
return self._nb_inserted_entities |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
616 |
@property |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
617 |
def nb_inserted_types(self): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
618 |
return self._nb_inserted_types |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
619 |
@property |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
620 |
def nb_inserted_relations(self): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
621 |
return self._nb_inserted_relations |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
622 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
623 |
def _put(self, type, item): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
624 |
raise RuntimeError('use create entity') |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
625 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
626 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
627 |
class MetaGenerator(object): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
628 |
def __init__(self, session, baseurl=None): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
629 |
self.session = session |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
630 |
self.source = session.repo.system_source |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
631 |
self.time = datetime.now() |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
632 |
if baseurl is None: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
633 |
config = session.vreg.config |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
634 |
baseurl = config['base-url'] or config.default_base_url() |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
635 |
if not baseurl[-1] == '/': |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
636 |
baseurl += '/' |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
637 |
self.baseurl = baseurl |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
638 |
# attributes/relations shared by all entities of the same type |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
639 |
self.etype_attrs = [] |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
640 |
self.etype_rels = [] |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
641 |
# attributes/relations specific to each entity |
5054
cb066d29166a
fix dataimport for 3.7.2
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
642 |
self.entity_attrs = ['cwuri'] |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
643 |
#self.entity_rels = [] XXX not handled (YAGNI?) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
644 |
schema = session.vreg.schema |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
645 |
rschema = schema.rschema |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
646 |
for rtype in META_RTYPES: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
647 |
if rtype in ('eid', 'cwuri') or rtype in VIRTUAL_RTYPES: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
648 |
continue |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
649 |
if rschema(rtype).final: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
650 |
self.etype_attrs.append(rtype) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
651 |
else: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
652 |
self.etype_rels.append(rtype) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
653 |
if not schema._eid_index: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
654 |
# test schema loaded from the fs |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
655 |
self.gen_is = self.test_gen_is |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
656 |
self.gen_is_instance_of = self.test_gen_is_instanceof |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
657 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
658 |
@cached |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
659 |
def base_etype_dicts(self, etype): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
660 |
entity = self.session.vreg['etypes'].etype_class(etype)(self.session) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
661 |
# entity are "surface" copied, avoid shared dict between copies |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
662 |
del entity.cw_extra_kwargs |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
663 |
for attr in self.etype_attrs: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
664 |
entity[attr] = self.generate(entity, attr) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
665 |
rels = {} |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
666 |
for rel in self.etype_rels: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
667 |
rels[rel] = self.generate(entity, rel) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
668 |
return entity, rels |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
669 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
670 |
def init_entity(self, entity): |
5054
cb066d29166a
fix dataimport for 3.7.2
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
671 |
entity.eid = self.source.create_eid(self.session) |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
672 |
for attr in self.entity_attrs: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
673 |
entity[attr] = self.generate(entity, attr) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
674 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
675 |
def generate(self, entity, rtype): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
676 |
return getattr(self, 'gen_%s' % rtype)(entity) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
677 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
678 |
def gen_cwuri(self, entity): |
5054
cb066d29166a
fix dataimport for 3.7.2
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
679 |
return u'%seid/%s' % (self.baseurl, entity.eid) |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
680 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
681 |
def gen_creation_date(self, entity): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
682 |
return self.time |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
683 |
def gen_modification_date(self, entity): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
684 |
return self.time |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
685 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
686 |
def gen_is(self, entity): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
687 |
return entity.e_schema.eid |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
688 |
def gen_is_instance_of(self, entity): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
689 |
eids = [] |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
690 |
for etype in entity.e_schema.ancestors() + [entity.e_schema]: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
691 |
eids.append(entity.e_schema.eid) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
692 |
return eids |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
693 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
694 |
def gen_created_by(self, entity): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
695 |
return self.session.user.eid |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
696 |
def gen_owned_by(self, entity): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
697 |
return self.session.user.eid |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
698 |
|
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
699 |
# implementations of gen_is / gen_is_instance_of to use during test where |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
700 |
# schema has been loaded from the fs (hence entity type schema eids are not |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
701 |
# known) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
702 |
def test_gen_is(self, entity): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
703 |
return eschema_eid(self.session, entity.e_schema) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
704 |
def test_gen_is_instanceof(self, entity): |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
705 |
eids = [] |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
706 |
for eschema in entity.e_schema.ancestors() + [entity.e_schema]: |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
707 |
eids.append(eschema_eid(self.session, eschema)) |
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4734
diff
changeset
|
708 |
return eids |