author | Nicolas Chauvat <nicolas.chauvat@logilab.fr> |
Sat, 18 Jun 2011 23:27:23 +0200 | |
branch | stable |
changeset 7537 | 1af162bd78b8 |
parent 7480 | 97a1768bbbaa |
child 7481 | 23ae090fc6a4 |
permissions | -rw-r--r-- |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
1 |
# copyright 2010-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
2 |
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
3 |
# |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
4 |
# This file is part of CubicWeb. |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
5 |
# |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
6 |
# CubicWeb is free software: you can redistribute it and/or modify it under the |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
7 |
# terms of the GNU Lesser General Public License as published by the Free |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
8 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
9 |
# any later version. |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
10 |
# |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
11 |
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
12 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
13 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
14 |
# details. |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
15 |
# |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
16 |
# You should have received a copy of the GNU Lesser General Public License along |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
17 |
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
18 |
"""datafeed parser for xml generated by cubicweb |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
19 |
|
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
20 |
Example of mapping for CWEntityXMLParser:: |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
21 |
|
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
22 |
{u'CWUser': { # EntityType |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
23 |
(u'in_group', u'subject', u'link'): [ # (rtype, role, action) |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
24 |
(u'CWGroup', {u'linkattr': u'name'})], # -> rules = [(EntityType, options), ...] |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
25 |
(u'tags', u'object', u'link-or-create'): [ # (...) |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
26 |
(u'Tag', {u'linkattr': u'name'})], # -> ... |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
27 |
(u'use_email', u'subject', u'copy'): [ # (...) |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
28 |
(u'EmailAddress', {})] # -> ... |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
29 |
} |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
30 |
} |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
31 |
|
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
32 |
""" |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
33 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
34 |
import urllib2 |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
35 |
import StringIO |
6963
5774d4ba4306
[datafeed] introduce a host mapping so dev instance may transparently redirect request to another host than the actual's one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6960
diff
changeset
|
36 |
import os.path as osp |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
37 |
from cookielib import CookieJar |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
38 |
from datetime import datetime, timedelta |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
39 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
40 |
from lxml import etree |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
41 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
42 |
from logilab.common.date import todate, totime |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
43 |
from logilab.common.textutils import splitstrip, text_to_dict |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
44 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
45 |
from yams.constraints import BASE_CONVERTERS |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
46 |
from yams.schema import role_name as rn |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
47 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
48 |
from cubicweb import ValidationError, typed_eid |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
49 |
from cubicweb.server.sources import datafeed |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
50 |
|
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
51 |
def ensure_str_keys(dic): |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
52 |
for key in dic: |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
53 |
dic[str(key)] = dic.pop(key) |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
54 |
|
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
55 |
# XXX see cubicweb.cwvreg.YAMS_TO_PY |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
56 |
# XXX see cubicweb.web.views.xmlrss.SERIALIZERS |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
57 |
DEFAULT_CONVERTERS = BASE_CONVERTERS.copy() |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
58 |
DEFAULT_CONVERTERS['String'] = unicode |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
59 |
DEFAULT_CONVERTERS['Password'] = lambda x: x.encode('utf8') |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
60 |
def convert_date(ustr): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
61 |
return todate(datetime.strptime(ustr, '%Y-%m-%d')) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
62 |
DEFAULT_CONVERTERS['Date'] = convert_date |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
63 |
def convert_datetime(ustr): |
7002
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
64 |
if '.' in ustr: # assume %Y-%m-%d %H:%M:%S.mmmmmm |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
65 |
ustr = ustr.split('.',1)[0] |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
66 |
return datetime.strptime(ustr, '%Y-%m-%d %H:%M:%S') |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
67 |
DEFAULT_CONVERTERS['Datetime'] = convert_datetime |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
68 |
def convert_time(ustr): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
69 |
return totime(datetime.strptime(ustr, '%H:%M:%S')) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
70 |
DEFAULT_CONVERTERS['Time'] = convert_time |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
71 |
def convert_interval(ustr): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
72 |
return time(seconds=int(ustr)) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
73 |
DEFAULT_CONVERTERS['Interval'] = convert_interval |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
74 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
75 |
# use a cookie enabled opener to use session cookie if any |
6994
4b2c7e401065
[sobjects/parsers] add gssapi auth handler to urllib2 opener
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6975
diff
changeset
|
76 |
_OPENER = urllib2.build_opener() |
6995
6efb5a5696bd
[datafeed] don't crash if python-kerberos isn't installed
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6994
diff
changeset
|
77 |
try: |
6efb5a5696bd
[datafeed] don't crash if python-kerberos isn't installed
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6994
diff
changeset
|
78 |
from logilab.common import urllib2ext |
6efb5a5696bd
[datafeed] don't crash if python-kerberos isn't installed
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6994
diff
changeset
|
79 |
_OPENER.add_handler(urllib2ext.HTTPGssapiAuthHandler()) |
6efb5a5696bd
[datafeed] don't crash if python-kerberos isn't installed
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6994
diff
changeset
|
80 |
except ImportError: # python-kerberos not available |
6efb5a5696bd
[datafeed] don't crash if python-kerberos isn't installed
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6994
diff
changeset
|
81 |
pass |
6994
4b2c7e401065
[sobjects/parsers] add gssapi auth handler to urllib2 opener
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6975
diff
changeset
|
82 |
_OPENER.add_handler(urllib2.HTTPCookieProcessor(CookieJar())) |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
83 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
84 |
def extract_typed_attrs(eschema, stringdict, converters=DEFAULT_CONVERTERS): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
85 |
typeddict = {} |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
86 |
for rschema in eschema.subject_relations(): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
87 |
if rschema.final and rschema in stringdict: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
88 |
if rschema == 'eid': |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
89 |
continue |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
90 |
attrtype = eschema.destination(rschema) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
91 |
typeddict[rschema.type] = converters[attrtype](stringdict[rschema]) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
92 |
return typeddict |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
93 |
|
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
94 |
def _parse_entity_etree(parent): |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
95 |
for node in list(parent): |
7002
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
96 |
try: |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
97 |
item = {'cwtype': unicode(node.tag), |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
98 |
'cwuri': node.attrib['cwuri'], |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
99 |
'eid': typed_eid(node.attrib['eid']), |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
100 |
} |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
101 |
except KeyError: |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
102 |
# cw < 3.11 compat mode XXX |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
103 |
item = {'cwtype': unicode(node.tag), |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
104 |
'cwuri': node.find('cwuri').text, |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
105 |
'eid': typed_eid(node.find('eid').text), |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
106 |
} |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
107 |
rels = {} |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
108 |
for child in node: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
109 |
role = child.get('role') |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
110 |
if role: |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
111 |
# relation |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
112 |
related = rels.setdefault(role, {}).setdefault(child.tag, []) |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
113 |
related += [ritem for ritem, _ in _parse_entity_etree(child)] |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
114 |
else: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
115 |
# attribute |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
116 |
item[child.tag] = unicode(child.text) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
117 |
yield item, rels |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
118 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
119 |
def build_search_rql(etype, attrs): |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
120 |
restrictions = ['X %(attr)s %%(%(attr)s)s'%{'attr': attr} for attr in attrs] |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
121 |
return 'Any X WHERE X is %s, %s' % (etype, ', '.join(restrictions)) |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
122 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
123 |
def rtype_role_rql(rtype, role): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
124 |
if role == 'object': |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
125 |
return 'Y %s X WHERE X eid %%(x)s' % rtype |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
126 |
else: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
127 |
return 'X %s Y WHERE X eid %%(x)s' % rtype |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
128 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
129 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
130 |
def _check_no_option(action, options, eid, _): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
131 |
if options: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
132 |
msg = _("'%s' action doesn't take any options") % action |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
133 |
raise ValidationError(eid, {rn('options', 'subject'): msg}) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
134 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
135 |
def _check_linkattr_option(action, options, eid, _): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
136 |
if not 'linkattr' in options: |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
137 |
msg = _("'%s' action requires 'linkattr' option") % action |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
138 |
raise ValidationError(eid, {rn('options', 'subject'): msg}) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
139 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
140 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
141 |
class CWEntityXMLParser(datafeed.DataFeedParser): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
142 |
"""datafeed parser for the 'xml' entity view""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
143 |
__regid__ = 'cw.entityxml' |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
144 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
145 |
action_options = { |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
146 |
'copy': _check_no_option, |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
147 |
'link-or-create': _check_linkattr_option, |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
148 |
'link': _check_linkattr_option, |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
149 |
} |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
150 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
151 |
def __init__(self, *args, **kwargs): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
152 |
super(CWEntityXMLParser, self).__init__(*args, **kwargs) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
153 |
self.action_methods = { |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
154 |
'copy': self.related_copy, |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
155 |
'link-or-create': self.related_link_or_create, |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
156 |
'link': self.related_link, |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
157 |
} |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
158 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
159 |
# mapping handling ######################################################### |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
160 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
161 |
def add_schema_config(self, schemacfg, checkonly=False): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
162 |
"""added CWSourceSchemaConfig, modify mapping accordingly""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
163 |
_ = self._cw._ |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
164 |
try: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
165 |
rtype = schemacfg.schema.rtype.name |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
166 |
except AttributeError: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
167 |
msg = _("entity and relation types can't be mapped, only attributes " |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
168 |
"or relations") |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
169 |
raise ValidationError(schemacfg.eid, {rn('cw_for_schema', 'subject'): msg}) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
170 |
if schemacfg.options: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
171 |
options = text_to_dict(schemacfg.options) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
172 |
else: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
173 |
options = {} |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
174 |
try: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
175 |
role = options.pop('role') |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
176 |
if role not in ('subject', 'object'): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
177 |
raise KeyError |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
178 |
except KeyError: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
179 |
msg = _('"role=subject" or "role=object" must be specified in options') |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
180 |
raise ValidationError(schemacfg.eid, {rn('options', 'subject'): msg}) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
181 |
try: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
182 |
action = options.pop('action') |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
183 |
self.action_options[action](action, options, schemacfg.eid, _) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
184 |
except KeyError: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
185 |
msg = _('"action" must be specified in options; allowed values are ' |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
186 |
'%s') % ', '.join(self.action_methods) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
187 |
raise ValidationError(schemacfg.eid, {rn('options', 'subject'): msg}) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
188 |
if not checkonly: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
189 |
if role == 'subject': |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
190 |
etype = schemacfg.schema.stype.name |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
191 |
ttype = schemacfg.schema.otype.name |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
192 |
else: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
193 |
etype = schemacfg.schema.otype.name |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
194 |
ttype = schemacfg.schema.stype.name |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
195 |
etyperules = self.source.mapping.setdefault(etype, {}) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
196 |
etyperules.setdefault((rtype, role, action), []).append( |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
197 |
(ttype, options) ) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
198 |
self.source.mapping_idx[schemacfg.eid] = ( |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
199 |
etype, rtype, role, action, ttype) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
200 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
201 |
def del_schema_config(self, schemacfg, checkonly=False): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
202 |
"""deleted CWSourceSchemaConfig, modify mapping accordingly""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
203 |
etype, rtype, role, action, ttype = self.source.mapping_idx[schemacfg.eid] |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
204 |
rules = self.source.mapping[etype][(rtype, role, action)] |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
205 |
rules = [x for x in rules if not x[0] == ttype] |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
206 |
if not rules: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
207 |
del self.source.mapping[etype][(rtype, role, action)] |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
208 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
209 |
# import handling ########################################################## |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
210 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
211 |
def process(self, url, partialcommit=True): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
212 |
"""IDataFeedParser main entry point""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
213 |
# XXX suppression support according to source configuration. If set, get |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
214 |
# all cwuri of entities from this source, and compare with newly |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
215 |
# imported ones |
7001
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
216 |
error = False |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
217 |
for item, rels in self.parse(url): |
7001
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
218 |
cwuri = item['cwuri'] |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
219 |
try: |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
220 |
self.process_item(item, rels) |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
221 |
if partialcommit: |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
222 |
# commit+set_pool instead of commit(reset_pool=False) to let |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
223 |
# other a chance to get our pool |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
224 |
self._cw.commit() |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
225 |
self._cw.set_pool() |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
226 |
except ValidationError, exc: |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
227 |
if partialcommit: |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
228 |
self.source.error('Skipping %s because of validation error %s' % (cwuri, exc)) |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
229 |
self._cw.rollback() |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
230 |
self._cw.set_pool() |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
231 |
error = True |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
232 |
else: |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
233 |
raise |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
234 |
return error |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
235 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
236 |
def parse(self, url): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
237 |
if not url.startswith('http'): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
238 |
stream = StringIO.StringIO(url) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
239 |
else: |
6963
5774d4ba4306
[datafeed] introduce a host mapping so dev instance may transparently redirect request to another host than the actual's one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6960
diff
changeset
|
240 |
for mappedurl in HOST_MAPPING: |
5774d4ba4306
[datafeed] introduce a host mapping so dev instance may transparently redirect request to another host than the actual's one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6960
diff
changeset
|
241 |
if url.startswith(mappedurl): |
5774d4ba4306
[datafeed] introduce a host mapping so dev instance may transparently redirect request to another host than the actual's one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6960
diff
changeset
|
242 |
url = url.replace(mappedurl, HOST_MAPPING[mappedurl], 1) |
5774d4ba4306
[datafeed] introduce a host mapping so dev instance may transparently redirect request to another host than the actual's one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6960
diff
changeset
|
243 |
break |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
244 |
self.source.info('GET %s', url) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
245 |
stream = _OPENER.open(url) |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
246 |
return _parse_entity_etree(etree.parse(stream).getroot()) |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
247 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
248 |
def process_item(self, item, rels): |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
249 |
entity = self.extid2entity(str(item.pop('cwuri')), item.pop('cwtype'), |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
250 |
item=item) |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
251 |
if not (self.created_during_pull(entity) or self.updated_during_pull(entity)): |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
252 |
self.notify_updated(entity) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
253 |
item.pop('eid') |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
254 |
# XXX check modification date |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
255 |
attrs = extract_typed_attrs(entity.e_schema, item) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
256 |
entity.set_attributes(**attrs) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
257 |
for (rtype, role, action), rules in self.source.mapping.get(entity.__regid__, {}).iteritems(): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
258 |
try: |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
259 |
related_items = rels[role][rtype] |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
260 |
except KeyError: |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
261 |
self.source.error('relation %s-%s not found in xml export of %s', |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
262 |
rtype, role, entity.__regid__) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
263 |
continue |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
264 |
try: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
265 |
actionmethod = self.action_methods[action] |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
266 |
except KeyError: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
267 |
raise Exception('Unknown action %s' % action) |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
268 |
actionmethod(entity, rtype, role, related_items, rules) |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
269 |
return entity |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
270 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
271 |
def before_entity_copy(self, entity, sourceparams): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
272 |
"""IDataFeedParser callback""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
273 |
attrs = extract_typed_attrs(entity.e_schema, sourceparams['item']) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
274 |
entity.cw_edited.update(attrs) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
275 |
|
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
276 |
def related_copy(self, entity, rtype, role, others, rules): |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
277 |
"""implementation of 'copy' action |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
278 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
279 |
Takes no option. |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
280 |
""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
281 |
assert not any(x[1] for x in rules), "'copy' action takes no option" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
282 |
ttypes = set([x[0] for x in rules]) |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
283 |
others = [item for item in others if item['cwtype'] in ttypes] |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
284 |
eids = [] # local eids |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
285 |
if not others: |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
286 |
self._clear_relation(entity, rtype, role, ttypes) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
287 |
return |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
288 |
for item in others: |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
289 |
item, _rels = self._complete_item(item) |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
290 |
other_entity = self.process_item(item, []) |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
291 |
eids.append(other_entity.eid) |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
292 |
self._set_relation(entity, rtype, role, eids) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
293 |
|
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
294 |
def related_link(self, entity, rtype, role, others, rules): |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
295 |
"""implementation of 'link' action |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
296 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
297 |
requires an options to control search of the linked entity. |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
298 |
""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
299 |
for ttype, options in rules: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
300 |
assert 'linkattr' in options, ( |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
301 |
"'link' action requires a list of attributes used to " |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
302 |
"search if the entity already exists") |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
303 |
self._related_link(entity, rtype, role, ttype, others, [options['linkattr']], |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
304 |
create_when_not_found=False) |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
305 |
|
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
306 |
def related_link_or_create(self, entity, rtype, role, others, rules): |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
307 |
"""implementation of 'link-or-create' action |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
308 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
309 |
requires an options to control search of the linked entity. |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
310 |
""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
311 |
for ttype, options in rules: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
312 |
assert 'linkattr' in options, ( |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
313 |
"'link-or-create' action requires a list of attributes used to " |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
314 |
"search if the entity already exists") |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
315 |
self._related_link(entity, rtype, role, ttype, others, [options['linkattr']], |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
316 |
create_when_not_found=True) |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
317 |
|
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
318 |
def _related_link(self, entity, rtype, role, ttype, others, searchattrs, |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
319 |
create_when_not_found): |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
320 |
def issubset(x,y): |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
321 |
return all(z in y for z in x) |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
322 |
eids = [] # local eids |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
323 |
for item in others: |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
324 |
if item['cwtype'] != ttype: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
325 |
continue |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
326 |
if not issubset(searchattrs, item): |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
327 |
item, _rels = self._complete_item(item, False) |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
328 |
if not issubset(searchattrs, item): |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
329 |
self.source.error('missing attribute, got %s expected keys %s' |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
330 |
% item, searchattrs) |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
331 |
continue |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
332 |
kwargs = dict((attr, item[attr]) for attr in searchattrs) |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
333 |
rql = build_search_rql(item['cwtype'], kwargs) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
334 |
rset = self._cw.execute(rql, kwargs) |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
335 |
if len(rset) > 1: |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
336 |
self.source.error('ambiguous link: found %s entity %s with attributes %s', |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
337 |
len(rset), item['cwtype'], kwargs) |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
338 |
elif len(rset) == 1: |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
339 |
eids.append(rset[0][0]) |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
340 |
elif create_when_not_found: |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
341 |
ensure_str_keys(kwargs) # XXX necessary with python < 2.6 |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
342 |
eids.append(self._cw.create_entity(item['cwtype'], **kwargs).eid) |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
343 |
else: |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
344 |
self.source.error('can not find %s entity with attributes %s', |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
345 |
item['cwtype'], kwargs) |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
346 |
if not eids: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
347 |
self._clear_relation(entity, rtype, role, (ttype,)) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
348 |
else: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
349 |
self._set_relation(entity, rtype, role, eids) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
350 |
|
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
351 |
def _complete_item(self, item, add_relations=True): |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
352 |
itemurl = item['cwuri'] + '?vid=xml' |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
353 |
if add_relations: |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
354 |
for rtype, role, _ in self.source.mapping.get(item['cwtype'], ()): |
7480
97a1768bbbaa
[parsers] format read by web/views/xmlrss is relation=%s-%s not %s_%s
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7354
diff
changeset
|
355 |
itemurl += '&relation=%s-%s' % (rtype, role) |
7354
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
356 |
item_rels = list(self.parse(itemurl)) |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
357 |
assert len(item_rels) == 1 |
f627ab500fda
[parsers] various refactorings
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7351
diff
changeset
|
358 |
return item_rels[0] |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
359 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
360 |
def _clear_relation(self, entity, rtype, role, ttypes): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
361 |
if entity.eid not in self.stats['created']: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
362 |
if len(ttypes) > 1: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
363 |
typerestr = ', Y is IN(%s)' % ','.join(ttypes) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
364 |
else: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
365 |
typerestr = ', Y is %s' % ','.join(ttypes) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
366 |
self._cw.execute('DELETE ' + rtype_role_rql(rtype, role) + typerestr, |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
367 |
{'x': entity.eid}) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
368 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
369 |
def _set_relation(self, entity, rtype, role, eids): |
7351
ed66f236715d
fix _set_relation when no target eids, update datafeed source pull_data arguments to raise on error during tests
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7002
diff
changeset
|
370 |
rqlbase = rtype_role_rql(rtype, role) |
ed66f236715d
fix _set_relation when no target eids, update datafeed source pull_data arguments to raise on error during tests
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7002
diff
changeset
|
371 |
rql = 'DELETE %s' % rqlbase |
ed66f236715d
fix _set_relation when no target eids, update datafeed source pull_data arguments to raise on error during tests
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7002
diff
changeset
|
372 |
if eids: |
ed66f236715d
fix _set_relation when no target eids, update datafeed source pull_data arguments to raise on error during tests
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7002
diff
changeset
|
373 |
eidstr = ','.join(str(eid) for eid in eids) |
ed66f236715d
fix _set_relation when no target eids, update datafeed source pull_data arguments to raise on error during tests
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7002
diff
changeset
|
374 |
rql += ', NOT Y eid IN (%s)' % eidstr |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
375 |
self._cw.execute(rql, {'x': entity.eid}) |
7351
ed66f236715d
fix _set_relation when no target eids, update datafeed source pull_data arguments to raise on error during tests
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7002
diff
changeset
|
376 |
if eids: |
ed66f236715d
fix _set_relation when no target eids, update datafeed source pull_data arguments to raise on error during tests
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7002
diff
changeset
|
377 |
if role == 'object': |
ed66f236715d
fix _set_relation when no target eids, update datafeed source pull_data arguments to raise on error during tests
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7002
diff
changeset
|
378 |
rql = 'SET %s, Y eid IN (%s), NOT Y %s X' % (rqlbase, eidstr, rtype) |
ed66f236715d
fix _set_relation when no target eids, update datafeed source pull_data arguments to raise on error during tests
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7002
diff
changeset
|
379 |
else: |
ed66f236715d
fix _set_relation when no target eids, update datafeed source pull_data arguments to raise on error during tests
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7002
diff
changeset
|
380 |
rql = 'SET %s, Y eid IN (%s), NOT X %s Y' % (rqlbase, eidstr, rtype) |
ed66f236715d
fix _set_relation when no target eids, update datafeed source pull_data arguments to raise on error during tests
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7002
diff
changeset
|
381 |
self._cw.execute(rql, {'x': entity.eid}) |
6963
5774d4ba4306
[datafeed] introduce a host mapping so dev instance may transparently redirect request to another host than the actual's one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6960
diff
changeset
|
382 |
|
5774d4ba4306
[datafeed] introduce a host mapping so dev instance may transparently redirect request to another host than the actual's one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6960
diff
changeset
|
383 |
def registration_callback(vreg): |
6970
a6ccbfbacf3d
[parser host mapping] take care to cases where apphome is None (c-c i18ncube command for instance)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6963
diff
changeset
|
384 |
vreg.register_all(globals().values(), __name__) |
6963
5774d4ba4306
[datafeed] introduce a host mapping so dev instance may transparently redirect request to another host than the actual's one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6960
diff
changeset
|
385 |
global HOST_MAPPING |
6970
a6ccbfbacf3d
[parser host mapping] take care to cases where apphome is None (c-c i18ncube command for instance)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6963
diff
changeset
|
386 |
HOST_MAPPING = {} |
a6ccbfbacf3d
[parser host mapping] take care to cases where apphome is None (c-c i18ncube command for instance)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6963
diff
changeset
|
387 |
if vreg.config.apphome: |
a6ccbfbacf3d
[parser host mapping] take care to cases where apphome is None (c-c i18ncube command for instance)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6963
diff
changeset
|
388 |
host_mapping_file = osp.join(vreg.config.apphome, 'hostmapping.py') |
a6ccbfbacf3d
[parser host mapping] take care to cases where apphome is None (c-c i18ncube command for instance)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6963
diff
changeset
|
389 |
if osp.exists(host_mapping_file): |
a6ccbfbacf3d
[parser host mapping] take care to cases where apphome is None (c-c i18ncube command for instance)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6963
diff
changeset
|
390 |
HOST_MAPPING = eval(file(host_mapping_file).read()) |
a6ccbfbacf3d
[parser host mapping] take care to cases where apphome is None (c-c i18ncube command for instance)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6963
diff
changeset
|
391 |
vreg.info('using host mapping %s from %s', HOST_MAPPING, host_mapping_file) |