author | Sylvain Thénault <sylvain.thenault@logilab.fr> |
Wed, 17 Feb 2010 16:42:52 +0100 | |
branch | stable |
changeset 4618 | 412a9f1f5fd0 |
parent 4613 | 141a4f613f8a |
child 4721 | 8f63691ccb7f |
permissions | -rw-r--r-- |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
1 |
# -*- coding: utf-8 -*- |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
2 |
"""This module provides tools to import tabular data. |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
3 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
4 |
:organization: Logilab |
4212
ab6573088b4a
update copyright: welcome 2010
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
3486
diff
changeset
|
5 |
:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
6 |
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
7 |
:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
8 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
9 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
10 |
Example of use (run this with `cubicweb-ctl shell instance import-script.py`): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
11 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
12 |
.. sourcecode:: python |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
13 |
|
3318
5b47b9f09bca
documentation : fixed docstring
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
3029
diff
changeset
|
14 |
from cubicweb.devtools.dataimport import * |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
15 |
# define data generators |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
16 |
GENERATORS = [] |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
17 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
18 |
USERS = [('Prenom', 'firstname', ()), |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
19 |
('Nom', 'surname', ()), |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
20 |
('Identifiant', 'login', ()), |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
21 |
] |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
22 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
23 |
def gen_users(ctl): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
24 |
for row in ctl.get_data('utilisateurs'): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
25 |
entity = mk_entity(row, USERS) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
26 |
entity['upassword'] = u'motdepasse' |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
27 |
ctl.check('login', entity['login'], None) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
28 |
ctl.store.add('CWUser', entity) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
29 |
email = {'address': row['email']} |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
30 |
ctl.store.add('EmailAddress', email) |
3003
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
31 |
ctl.store.relate(entity['eid'], 'use_email', email['eid']) |
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
32 |
ctl.store.rql('SET U in_group G WHERE G name "users", U eid %(x)s', {'x':entity['eid']}) |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
33 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
34 |
CHK = [('login', check_doubles, 'Utilisateurs Login', |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
35 |
'Deux utilisateurs ne devraient pas avoir le même login.'), |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
36 |
] |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
37 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
38 |
GENERATORS.append( (gen_users, CHK) ) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
39 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
40 |
# create controller |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
41 |
ctl = CWImportController(RQLObjectStore()) |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
42 |
ctl.askerror = 1 |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
43 |
ctl.generators = GENERATORS |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
44 |
ctl.store._checkpoint = checkpoint |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
45 |
ctl.store._rql = rql |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
46 |
ctl.data['utilisateurs'] = lazytable(utf8csvreader(open('users.csv'))) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
47 |
# run |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
48 |
ctl.run() |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
49 |
sys.exit(0) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
50 |
|
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
51 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
52 |
.. BUG fichier à une colonne pose un problème de parsing |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
53 |
.. TODO rollback() |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
54 |
""" |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
55 |
__docformat__ = "restructuredtext en" |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
56 |
|
4186
ca7e526b07b6
import cleanup, check data file exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4173
diff
changeset
|
57 |
import sys |
ca7e526b07b6
import cleanup, check data file exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4173
diff
changeset
|
58 |
import csv |
ca7e526b07b6
import cleanup, check data file exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4173
diff
changeset
|
59 |
import traceback |
ca7e526b07b6
import cleanup, check data file exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4173
diff
changeset
|
60 |
import os.path as osp |
ca7e526b07b6
import cleanup, check data file exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4173
diff
changeset
|
61 |
from StringIO import StringIO |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
62 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
63 |
from logilab.common import shellutils |
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
64 |
from logilab.common.deprecation import deprecated |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
65 |
|
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
66 |
def ucsvreader_pb(filepath, encoding='utf-8', separator=',', quote='"', |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
67 |
skipfirst=False, withpb=True): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
68 |
"""same as ucsvreader but a progress bar is displayed as we iter on rows""" |
4186
ca7e526b07b6
import cleanup, check data file exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4173
diff
changeset
|
69 |
if not osp.exists(filepath): |
ca7e526b07b6
import cleanup, check data file exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4173
diff
changeset
|
70 |
raise Exception("file doesn't exists: %s" % filepath) |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
71 |
rowcount = int(shellutils.Execute('wc -l "%s"' % filepath).out.strip().split()[0]) |
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
72 |
if skipfirst: |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
73 |
rowcount -= 1 |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
74 |
if withpb: |
4140
46ddd27a4ca4
tweaks output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4136
diff
changeset
|
75 |
pb = shellutils.ProgressBar(rowcount, 50) |
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
76 |
for urow in ucsvreader(file(filepath), encoding, separator, quote, skipfirst): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
77 |
yield urow |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
78 |
if withpb: |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
79 |
pb.update() |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
80 |
print ' %s rows imported' % rowcount |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
81 |
|
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
82 |
def ucsvreader(stream, encoding='utf-8', separator=',', quote='"', |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
83 |
skipfirst=False): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
84 |
"""A csv reader that accepts files with any encoding and outputs unicode |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
85 |
strings |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
86 |
""" |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
87 |
it = iter(csv.reader(stream, delimiter=separator, quotechar=quote)) |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
88 |
if skipfirst: |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
89 |
it.next() |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
90 |
for row in it: |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
91 |
yield [item.decode(encoding) for item in row] |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
92 |
|
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
93 |
utf8csvreader = deprecated('use ucsvreader instead')(ucsvreader) |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
94 |
|
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
95 |
def commit_every(nbit, store, it): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
96 |
for i, x in enumerate(it): |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
97 |
yield x |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
98 |
if nbit is not None and i % nbit: |
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
99 |
store.checkpoint() |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
100 |
if nbit is not None: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
101 |
store.checkpoint() |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
102 |
|
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
103 |
def lazytable(reader): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
104 |
"""The first row is taken to be the header of the table and |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
105 |
used to output a dict for each row of data. |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
106 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
107 |
>>> data = lazytable(utf8csvreader(open(filename))) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
108 |
""" |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
109 |
header = reader.next() |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
110 |
for row in reader: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
111 |
yield dict(zip(header, row)) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
112 |
|
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
113 |
def mk_entity(row, map): |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
114 |
"""Return a dict made from sanitized mapped values. |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
115 |
|
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
116 |
ValidationError can be raised on unexpected values found in checkers |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
117 |
|
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
118 |
>>> row = {'myname': u'dupont'} |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
119 |
>>> map = [('myname', u'name', (capitalize_if_unicase,))] |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
120 |
>>> mk_entity(row, map) |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
121 |
{'name': u'Dupont'} |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
122 |
>>> row = {'myname': u'dupont', 'optname': u''} |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
123 |
>>> map = [('myname', u'name', (capitalize_if_unicase,)), |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
124 |
... ('optname', u'MARKER', (optional,))] |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
125 |
>>> mk_entity(row, map) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
126 |
{'name': u'Dupont'} |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
127 |
""" |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
128 |
res = {} |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
129 |
assert isinstance(row, dict) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
130 |
assert isinstance(map, list) |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
131 |
for src, dest, funcs in map: |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
132 |
assert not (required in funcs and optional in funcs), "optional and required checks are exclusive" |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
133 |
res[dest] = row[src] |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
134 |
try: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
135 |
for func in funcs: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
136 |
res[dest] = func(res[dest]) |
4613
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
137 |
if res[dest] is None: |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
138 |
raise AssertionError('undetermined value') |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
139 |
except AssertionError, err: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
140 |
if optional in funcs: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
141 |
# Forget this field if exception is coming from optional function |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
142 |
del res[dest] |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
143 |
else: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
144 |
raise AssertionError('error with "%s" field: %s' % (src, err)) |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
145 |
return res |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
146 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
147 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
148 |
# user interactions ############################################################ |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
149 |
|
3029
bc573d5fb5b7
F [devtools] by default dataimport prints message on stdout
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
3003
diff
changeset
|
150 |
def tell(msg): |
bc573d5fb5b7
F [devtools] by default dataimport prints message on stdout
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
3003
diff
changeset
|
151 |
print msg |
bc573d5fb5b7
F [devtools] by default dataimport prints message on stdout
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
3003
diff
changeset
|
152 |
|
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
153 |
def confirm(question): |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
154 |
"""A confirm function that asks for yes/no/abort and exits on abort.""" |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
155 |
answer = shellutils.ASK.ask(question, ('Y','n','abort'), 'Y') |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
156 |
if answer == 'abort': |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
157 |
sys.exit(1) |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
158 |
return answer == 'Y' |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
159 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
160 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
161 |
class catch_error(object): |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
162 |
"""Helper for @contextmanager decorator.""" |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
163 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
164 |
def __init__(self, ctl, key='unexpected error', msg=None): |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
165 |
self.ctl = ctl |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
166 |
self.key = key |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
167 |
self.msg = msg |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
168 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
169 |
def __enter__(self): |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
170 |
return self |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
171 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
172 |
def __exit__(self, type, value, traceback): |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
173 |
if type is not None: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
174 |
if issubclass(type, (KeyboardInterrupt, SystemExit)): |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
175 |
return # re-raise |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
176 |
if self.ctl.catcherrors: |
4173
cfd5d3270f99
msg isn't defined there, but we've to give traceback information to record error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4152
diff
changeset
|
177 |
self.ctl.record_error(self.key, None, type, value, traceback) |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
178 |
return True # silent |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
179 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
180 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
181 |
# base sanitizing functions #################################################### |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
182 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
183 |
def capitalize_if_unicase(txt): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
184 |
if txt.isupper() or txt.islower(): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
185 |
return txt.capitalize() |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
186 |
return txt |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
187 |
|
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
188 |
def uppercase(txt): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
189 |
return txt.upper() |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
190 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
191 |
def lowercase(txt): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
192 |
return txt.lower() |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
193 |
|
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
194 |
def no_space(txt): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
195 |
return txt.replace(' ','') |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
196 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
197 |
def no_uspace(txt): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
198 |
return txt.replace(u'\xa0','') |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
199 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
200 |
def no_dash(txt): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
201 |
return txt.replace('-','') |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
202 |
|
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
203 |
def decimal(value): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
204 |
"""cast to float but with comma replacement |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
205 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
206 |
We take care of some locale format as replacing ',' by '.'""" |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
207 |
value = value.replace(',', '.') |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
208 |
try: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
209 |
return float(value) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
210 |
except Exception, err: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
211 |
raise AssertionError(err) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
212 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
213 |
def integer(value): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
214 |
try: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
215 |
return int(value) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
216 |
except Exception, err: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
217 |
raise AssertionError(err) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
218 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
219 |
def strip(txt): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
220 |
return txt.strip() |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
221 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
222 |
def yesno(value): |
4613
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
223 |
"""simple heuristic that returns boolean value |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
224 |
|
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
225 |
>>> yesno("Yes") |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
226 |
True |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
227 |
>>> yesno("oui") |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
228 |
True |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
229 |
>>> yesno("1") |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
230 |
True |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
231 |
>>> yesno("11") |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
232 |
True |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
233 |
>>> yesno("") |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
234 |
False |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
235 |
>>> yesno("Non") |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
236 |
False |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
237 |
>>> yesno("blablabla") |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
238 |
False |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
239 |
""" |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
240 |
if value: |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
241 |
return value.lower()[0] in 'yo1' |
141a4f613f8a
[R] dataimport: refine error detection
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4527
diff
changeset
|
242 |
return False |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
243 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
244 |
def isalpha(value): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
245 |
if value.isalpha(): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
246 |
return value |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
247 |
raise AssertionError("not all characters in the string alphabetic") |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
248 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
249 |
def optional(value): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
250 |
"""validation error will not been raised if you add this checker in chain""" |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
251 |
return value |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
252 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
253 |
def required(value): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
254 |
"""raise AssertionError is value is empty |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
255 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
256 |
This check should be often found in last position in the chain. |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
257 |
""" |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
258 |
if bool(value): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
259 |
return value |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
260 |
raise AssertionError("required") |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
261 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
262 |
@deprecated('use required(value)') |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
263 |
def nonempty(value): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
264 |
return required(value) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
265 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
266 |
@deprecated('use integer(value)') |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
267 |
def alldigits(txt): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
268 |
if txt.isdigit(): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
269 |
return txt |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
270 |
else: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
271 |
return u'' |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
272 |
|
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
273 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
274 |
# base integrity checking functions ############################################ |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
275 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
276 |
def check_doubles(buckets): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
277 |
"""Extract the keys that have more than one item in their bucket.""" |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
278 |
return [(key, len(value)) for key,value in buckets.items() if len(value) > 1] |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
279 |
|
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
280 |
def check_doubles_not_none(buckets): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
281 |
"""Extract the keys that have more than one item in their bucket.""" |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
282 |
return [(key, len(value)) for key,value in buckets.items() if key is not None and len(value) > 1] |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
283 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
284 |
|
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
285 |
# object stores ################################################################# |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
286 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
287 |
class ObjectStore(object): |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
288 |
"""Store objects in memory for *faster* validation (development mode) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
289 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
290 |
But it will not enforce the constraints of the schema and hence will miss some problems |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
291 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
292 |
>>> store = ObjectStore() |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
293 |
>>> user = {'login': 'johndoe'} |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
294 |
>>> store.add('CWUser', user) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
295 |
>>> group = {'name': 'unknown'} |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
296 |
>>> store.add('CWUser', group) |
3003
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
297 |
>>> store.relate(user['eid'], 'in_group', group['eid']) |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
298 |
""" |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
299 |
def __init__(self): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
300 |
self.items = [] |
3003
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
301 |
self.eids = {} |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
302 |
self.types = {} |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
303 |
self.relations = set() |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
304 |
self.indexes = {} |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
305 |
self._rql = None |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
306 |
self._checkpoint = None |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
307 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
308 |
def _put(self, type, item): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
309 |
self.items.append(item) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
310 |
return len(self.items) - 1 |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
311 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
312 |
def add(self, type, item): |
3486
ea6bf6f9ba0c
[cwctl] improve dialog messages
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
3318
diff
changeset
|
313 |
assert isinstance(item, dict), 'item is not a dict but a %s' % type(item) |
3003
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
314 |
eid = item['eid'] = self._put(type, item) |
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
315 |
self.eids[eid] = item |
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
316 |
self.types.setdefault(type, []).append(eid) |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
317 |
|
3003
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
318 |
def relate(self, eid_from, rtype, eid_to): |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
319 |
"""Add new relation (reverse type support is available) |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
320 |
|
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
321 |
>>> 1,2 = eid_from, eid_to |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
322 |
>>> self.relate(eid_from, 'in_group', eid_to) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
323 |
1, 'in_group', 2 |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
324 |
>>> self.relate(eid_from, 'reverse_in_group', eid_to) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
325 |
2, 'in_group', 1 |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
326 |
""" |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
327 |
if rtype.startswith('reverse_'): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
328 |
eid_from, eid_to = eid_to, eid_from |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
329 |
rtype = rtype[8:] |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
330 |
relation = eid_from, rtype, eid_to |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
331 |
self.relations.add(relation) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
332 |
return relation |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
333 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
334 |
def build_index(self, name, type, func=None): |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
335 |
index = {} |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
336 |
if func is None or not callable(func): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
337 |
func = lambda x: x['eid'] |
3003
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
338 |
for eid in self.types[type]: |
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
339 |
index.setdefault(func(self.eids[eid]), []).append(eid) |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
340 |
assert index, "new index '%s' cannot be empty" % name |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
341 |
self.indexes[name] = index |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
342 |
|
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
343 |
def build_rqlindex(self, name, type, key, rql, rql_params=False, func=None): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
344 |
"""build an index by rql query |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
345 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
346 |
rql should return eid in first column |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
347 |
ctl.store.build_index('index_name', 'users', 'login', 'Any U WHERE U is CWUser') |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
348 |
""" |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
349 |
rset = self.rql(rql, rql_params or {}) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
350 |
for entity in rset.entities(): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
351 |
getattr(entity, key) # autopopulate entity with key attribute |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
352 |
self.eids[entity.eid] = dict(entity) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
353 |
if entity.eid not in self.types.setdefault(type, []): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
354 |
self.types[type].append(entity.eid) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
355 |
assert self.types[type], "new index type '%s' cannot be empty (0 record found)" % type |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
356 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
357 |
# Build index with specified key |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
358 |
func = lambda x: x[key] |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
359 |
self.build_index(name, type, func) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
360 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
361 |
@deprecated('get_many() deprecated. Use fetch() instead') |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
362 |
def get_many(self, name, key): |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
363 |
return self.fetch(name, key, unique=False) |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
364 |
|
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
365 |
@deprecated('get_one() deprecated. Use fetch(..., unique=True) instead') |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
366 |
def get_one(self, name, key): |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
367 |
return self.fetch(name, key, unique=True) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
368 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
369 |
def fetch(self, name, key, unique=False, decorator=None): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
370 |
""" |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
371 |
decorator is a callable method or an iterator of callable methods (usually a lambda function) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
372 |
decorator=lambda x: x[:1] (first value is returned) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
373 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
374 |
We can use validation check function available in _entity |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
375 |
""" |
3003
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
376 |
eids = self.indexes[name].get(key, []) |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
377 |
if decorator is not None: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
378 |
if not hasattr(decorator, '__iter__'): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
379 |
decorator = (decorator,) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
380 |
for f in decorator: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
381 |
eids = f(eids) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
382 |
if unique: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
383 |
assert len(eids) == 1, u'expected a single one value for key "%s" in index "%s". Got %i' % (key, name, len(eids)) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
384 |
eids = eids[0] # FIXME maybe it's better to keep an iterator here ? |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
385 |
return eids |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
386 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
387 |
def find(self, type, key, value): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
388 |
for idx in self.types[type]: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
389 |
item = self.items[idx] |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
390 |
if item[key] == value: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
391 |
yield item |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
392 |
|
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
393 |
def rql(self, *args): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
394 |
if self._rql is not None: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
395 |
return self._rql(*args) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
396 |
|
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
397 |
def checkpoint(self): |
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
398 |
pass |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
399 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
400 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
401 |
class RQLObjectStore(ObjectStore): |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
402 |
"""ObjectStore that works with an actual RQL repository (production mode)""" |
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
403 |
_rql = None # bw compat |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
404 |
|
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
405 |
def __init__(self, session=None, checkpoint=None): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
406 |
ObjectStore.__init__(self) |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
407 |
if session is not None: |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
408 |
if not hasattr(session, 'set_pool'): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
409 |
# connection |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
410 |
cnx = session |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
411 |
session = session.request() |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
412 |
session.set_pool = lambda : None |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
413 |
checkpoint = checkpoint or cnx.commit |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
414 |
self.session = session |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
415 |
self.checkpoint = checkpoint or session.commit |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
416 |
elif checkpoint is not None: |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
417 |
self.checkpoint = checkpoint |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
418 |
|
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
419 |
def rql(self, *args): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
420 |
if self._rql is not None: |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
421 |
return self._rql(*args) |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
422 |
self.session.set_pool() |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
423 |
return self.session.execute(*args) |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
424 |
|
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
425 |
def create_entity(self, *args, **kwargs): |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
426 |
self.session.set_pool() |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
427 |
entity = self.session.create_entity(*args, **kwargs) |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
428 |
self.eids[entity.eid] = entity |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
429 |
self.types.setdefault(args[0], []).append(entity.eid) |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
430 |
return entity |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
431 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
432 |
def _put(self, type, item): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
433 |
query = ('INSERT %s X: ' % type) + ', '.join(['X %s %%(%s)s' % (key,key) for key in item]) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
434 |
return self.rql(query, item)[0][0] |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
435 |
|
3003
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
436 |
def relate(self, eid_from, rtype, eid_to): |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
437 |
# if reverse relation is found, eids are exchanged |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
438 |
eid_from, rtype, eid_to = super(RQLObjectStore, self).relate(eid_from, rtype, eid_to) |
4136
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
439 |
self.rql('SET X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype, |
47060a66c97f
dataimport refactoring / improvments, keeping bw compat (for now)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3486
diff
changeset
|
440 |
{'x': int(eid_from), 'y': int(eid_to)}, ('x', 'y')) |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
441 |
|
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
442 |
|
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
443 |
# the import controller ######################################################## |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
444 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
445 |
class CWImportController(object): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
446 |
"""Controller of the data import process. |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
447 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
448 |
>>> ctl = CWImportController(store) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
449 |
>>> ctl.generators = list_of_data_generators |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
450 |
>>> ctl.data = dict_of_data_tables |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
451 |
>>> ctl.run() |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
452 |
""" |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
453 |
|
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
454 |
def __init__(self, store, askerror=0, catcherrors=None, tell=tell, |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
455 |
commitevery=50): |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
456 |
self.store = store |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
457 |
self.generators = None |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
458 |
self.data = {} |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
459 |
self.errors = None |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
460 |
self.askerror = askerror |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
461 |
if catcherrors is None: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
462 |
catcherrors = askerror |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
463 |
self.catcherrors = catcherrors |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
464 |
self.commitevery = commitevery # set to None to do a single commit |
3029
bc573d5fb5b7
F [devtools] by default dataimport prints message on stdout
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
3003
diff
changeset
|
465 |
self._tell = tell |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
466 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
467 |
def check(self, type, key, value): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
468 |
self._checks.setdefault(type, {}).setdefault(key, []).append(value) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
469 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
470 |
def check_map(self, entity, key, map, default): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
471 |
try: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
472 |
entity[key] = map[entity[key]] |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
473 |
except KeyError: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
474 |
self.check(key, entity[key], None) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
475 |
entity[key] = default |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
476 |
|
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
477 |
def record_error(self, key, msg=None, type=None, value=None, tb=None): |
4186
ca7e526b07b6
import cleanup, check data file exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4173
diff
changeset
|
478 |
tmp = StringIO() |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
479 |
if type is None: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
480 |
traceback.print_exc(file=tmp) |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
481 |
else: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
482 |
traceback.print_exception(type, value, tb, file=tmp) |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
483 |
print tmp.getvalue() |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
484 |
# use a list to avoid counting a <nb lines> errors instead of one |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
485 |
errorlog = self.errors.setdefault(key, []) |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
486 |
if msg is None: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
487 |
errorlog.append(tmp.getvalue().splitlines()) |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
488 |
else: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
489 |
errorlog.append( (msg, tmp.getvalue().splitlines()) ) |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
490 |
|
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
491 |
def run(self): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
492 |
self.errors = {} |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
493 |
for func, checks in self.generators: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
494 |
self._checks = {} |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
495 |
func_name = func.__name__[4:] # XXX |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
496 |
self.tell("Import '%s'..." % func_name) |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
497 |
try: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
498 |
func(self) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
499 |
except: |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
500 |
if self.catcherrors: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
501 |
self.record_error(func_name, 'While calling %s' % func.__name__) |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
502 |
else: |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
503 |
raise |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
504 |
for key, func, title, help in checks: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
505 |
buckets = self._checks.get(key) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
506 |
if buckets: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
507 |
err = func(buckets) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
508 |
if err: |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
509 |
self.errors[title] = (help, err) |
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
510 |
self.store.checkpoint() |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
511 |
nberrors = sum(len(err[1]) for err in self.errors.values()) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
512 |
self.tell('\nImport completed: %i entities, %i types, %i relations and %i errors' |
3003
2944ee420dca
R [dataimport] rename uid to eid
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2974
diff
changeset
|
513 |
% (len(self.store.eids), len(self.store.types), |
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
514 |
len(self.store.relations), nberrors)) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
515 |
if self.errors: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
516 |
if self.askerror==2 or (self.askerror and confirm('Display errors ?')): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
517 |
from pprint import pformat |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
518 |
for errkey, error in self.errors.items(): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
519 |
self.tell("\n%s (%s): %d\n" % (error[0], errkey, len(error[1]))) |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
520 |
self.tell(pformat(sorted(error[1]))) |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
521 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
522 |
def get_data(self, key): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
523 |
return self.data.get(key) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
524 |
|
4527
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
525 |
def index(self, name, key, value, unique=False): |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
526 |
"""create a new index |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
527 |
|
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
528 |
If unique is set to True, only first occurence will be kept not the following ones |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
529 |
""" |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
530 |
if unique: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
531 |
try: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
532 |
if value in self.store.indexes[name][key]: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
533 |
return |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
534 |
except KeyError: |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
535 |
# we're sure that one is the first occurence; so continue... |
67ab70e98488
[R] devtools: improve default data import mechanism
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
4252
diff
changeset
|
536 |
pass |
2974
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
537 |
self.store.indexes.setdefault(name, {}).setdefault(key, []).append(value) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
538 |
|
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
539 |
def tell(self, msg): |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
540 |
self._tell(msg) |
3dfe497e5afa
F tools to import data
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
diff
changeset
|
541 |
|
4152
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
542 |
def iter_and_commit(self, datakey): |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
543 |
"""iter rows, triggering commit every self.commitevery iterations""" |
30fd1229137d
new catch_error context manager, nicer controller __init__ and new iter_and_commit(datakey) method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4140
diff
changeset
|
544 |
return commit_every(self.commitevery, self.store, self.get_data(datakey)) |