author | Sylvain Thénault <sylvain.thenault@logilab.fr> |
Thu, 17 Feb 2011 16:46:38 +0100 | |
changeset 7009 | 123947d4a875 |
parent 7002 | 29f085f6177b |
child 7351 | ed66f236715d |
permissions | -rw-r--r-- |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
1 |
# copyright 2010-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
2 |
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
3 |
# |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
4 |
# This file is part of CubicWeb. |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
5 |
# |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
6 |
# CubicWeb is free software: you can redistribute it and/or modify it under the |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
7 |
# terms of the GNU Lesser General Public License as published by the Free |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
8 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
9 |
# any later version. |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
10 |
# |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
11 |
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
12 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
13 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
14 |
# details. |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
15 |
# |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
16 |
# You should have received a copy of the GNU Lesser General Public License along |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
17 |
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
18 |
"""datafeed parser for xml generated by cubicweb""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
19 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
20 |
import urllib2 |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
21 |
import StringIO |
6963
5774d4ba4306
[datafeed] introduce a host mapping so dev instance may transparently redirect request to another host than the actual's one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6960
diff
changeset
|
22 |
import os.path as osp |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
23 |
from cookielib import CookieJar |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
24 |
from datetime import datetime, timedelta |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
25 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
26 |
from lxml import etree |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
27 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
28 |
from logilab.common.date import todate, totime |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
29 |
from logilab.common.textutils import splitstrip, text_to_dict |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
30 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
31 |
from yams.constraints import BASE_CONVERTERS |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
32 |
from yams.schema import role_name as rn |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
33 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
34 |
from cubicweb import ValidationError, typed_eid |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
35 |
from cubicweb.server.sources import datafeed |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
36 |
|
6975
75a232b2e477
[test] python 2.5 specifix
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6970
diff
changeset
|
37 |
def ensure_str_keys(dict): |
75a232b2e477
[test] python 2.5 specifix
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6970
diff
changeset
|
38 |
for key in dict: |
75a232b2e477
[test] python 2.5 specifix
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6970
diff
changeset
|
39 |
dict[str(key)] = dict.pop(key) |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
40 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
41 |
# see cubicweb.web.views.xmlrss.SERIALIZERS |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
42 |
DEFAULT_CONVERTERS = BASE_CONVERTERS.copy() |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
43 |
DEFAULT_CONVERTERS['String'] = unicode |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
44 |
DEFAULT_CONVERTERS['Password'] = lambda x: x.encode('utf8') |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
45 |
def convert_date(ustr): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
46 |
return todate(datetime.strptime(ustr, '%Y-%m-%d')) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
47 |
DEFAULT_CONVERTERS['Date'] = convert_date |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
48 |
def convert_datetime(ustr): |
7002
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
49 |
if '.' in ustr: # assume %Y-%m-%d %H:%M:%S.mmmmmm |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
50 |
ustr = ustr.split('.',1)[0] |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
51 |
return datetime.strptime(ustr, '%Y-%m-%d %H:%M:%S') |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
52 |
DEFAULT_CONVERTERS['Datetime'] = convert_datetime |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
53 |
def convert_time(ustr): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
54 |
return totime(datetime.strptime(ustr, '%H:%M:%S')) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
55 |
DEFAULT_CONVERTERS['Time'] = convert_time |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
56 |
def convert_interval(ustr): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
57 |
return time(seconds=int(ustr)) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
58 |
DEFAULT_CONVERTERS['Interval'] = convert_interval |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
59 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
60 |
# use a cookie enabled opener to use session cookie if any |
6994
4b2c7e401065
[sobjects/parsers] add gssapi auth handler to urllib2 opener
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6975
diff
changeset
|
61 |
_OPENER = urllib2.build_opener() |
6995
6efb5a5696bd
[datafeed] don't crash if python-kerberos isn't installed
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6994
diff
changeset
|
62 |
try: |
6efb5a5696bd
[datafeed] don't crash if python-kerberos isn't installed
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6994
diff
changeset
|
63 |
from logilab.common import urllib2ext |
6efb5a5696bd
[datafeed] don't crash if python-kerberos isn't installed
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6994
diff
changeset
|
64 |
_OPENER.add_handler(urllib2ext.HTTPGssapiAuthHandler()) |
6efb5a5696bd
[datafeed] don't crash if python-kerberos isn't installed
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6994
diff
changeset
|
65 |
except ImportError: # python-kerberos not available |
6efb5a5696bd
[datafeed] don't crash if python-kerberos isn't installed
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6994
diff
changeset
|
66 |
pass |
6994
4b2c7e401065
[sobjects/parsers] add gssapi auth handler to urllib2 opener
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6975
diff
changeset
|
67 |
_OPENER.add_handler(urllib2.HTTPCookieProcessor(CookieJar())) |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
68 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
69 |
def extract_typed_attrs(eschema, stringdict, converters=DEFAULT_CONVERTERS): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
70 |
typeddict = {} |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
71 |
for rschema in eschema.subject_relations(): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
72 |
if rschema.final and rschema in stringdict: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
73 |
if rschema == 'eid': |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
74 |
continue |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
75 |
attrtype = eschema.destination(rschema) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
76 |
typeddict[rschema.type] = converters[attrtype](stringdict[rschema]) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
77 |
return typeddict |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
78 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
79 |
def _entity_etree(parent): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
80 |
for node in list(parent): |
7002
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
81 |
try: |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
82 |
item = {'cwtype': unicode(node.tag), |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
83 |
'cwuri': node.attrib['cwuri'], |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
84 |
'eid': typed_eid(node.attrib['eid']), |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
85 |
} |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
86 |
except KeyError: |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
87 |
# cw < 3.11 compat mode XXX |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
88 |
item = {'cwtype': unicode(node.tag), |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
89 |
'cwuri': node.find('cwuri').text, |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
90 |
'eid': typed_eid(node.find('eid').text), |
29f085f6177b
[sobjects/parsers] compatibility with xml view of cw <= 3.10
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7001
diff
changeset
|
91 |
} |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
92 |
rels = {} |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
93 |
for child in node: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
94 |
role = child.get('role') |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
95 |
if child.get('role'): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
96 |
# relation |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
97 |
related = rels.setdefault(role, {}).setdefault(child.tag, []) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
98 |
related += [ritem for ritem, _ in _entity_etree(child)] |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
99 |
else: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
100 |
# attribute |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
101 |
item[child.tag] = unicode(child.text) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
102 |
yield item, rels |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
103 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
104 |
def build_search_rql(etype, attrs): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
105 |
restrictions = [] |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
106 |
for attr in attrs: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
107 |
restrictions.append('X %(attr)s %%(%(attr)s)s' % {'attr': attr}) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
108 |
return 'Any X WHERE X is %s, %s' % (etype, ','.join(restrictions)) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
109 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
110 |
def rtype_role_rql(rtype, role): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
111 |
if role == 'object': |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
112 |
return 'Y %s X WHERE X eid %%(x)s' % rtype |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
113 |
else: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
114 |
return 'X %s Y WHERE X eid %%(x)s' % rtype |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
115 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
116 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
117 |
def _check_no_option(action, options, eid, _): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
118 |
if options: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
119 |
msg = _("'%s' action doesn't take any options") % action |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
120 |
raise ValidationError(eid, {rn('options', 'subject'): msg}) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
121 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
122 |
def _check_linkattr_option(action, options, eid, _): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
123 |
if not 'linkattr' in options: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
124 |
msg = _("'%s' action require 'linkattr' option") % action |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
125 |
raise ValidationError(eid, {rn('options', 'subject'): msg}) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
126 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
127 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
128 |
class CWEntityXMLParser(datafeed.DataFeedParser): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
129 |
"""datafeed parser for the 'xml' entity view""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
130 |
__regid__ = 'cw.entityxml' |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
131 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
132 |
action_options = { |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
133 |
'copy': _check_no_option, |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
134 |
'link-or-create': _check_linkattr_option, |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
135 |
'link': _check_linkattr_option, |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
136 |
} |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
137 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
138 |
def __init__(self, *args, **kwargs): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
139 |
super(CWEntityXMLParser, self).__init__(*args, **kwargs) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
140 |
self.action_methods = { |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
141 |
'copy': self.related_copy, |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
142 |
'link-or-create': self.related_link_or_create, |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
143 |
'link': self.related_link, |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
144 |
} |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
145 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
146 |
# mapping handling ######################################################### |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
147 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
148 |
def add_schema_config(self, schemacfg, checkonly=False): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
149 |
"""added CWSourceSchemaConfig, modify mapping accordingly""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
150 |
_ = self._cw._ |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
151 |
try: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
152 |
rtype = schemacfg.schema.rtype.name |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
153 |
except AttributeError: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
154 |
msg = _("entity and relation types can't be mapped, only attributes " |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
155 |
"or relations") |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
156 |
raise ValidationError(schemacfg.eid, {rn('cw_for_schema', 'subject'): msg}) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
157 |
if schemacfg.options: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
158 |
options = text_to_dict(schemacfg.options) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
159 |
else: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
160 |
options = {} |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
161 |
try: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
162 |
role = options.pop('role') |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
163 |
if role not in ('subject', 'object'): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
164 |
raise KeyError |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
165 |
except KeyError: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
166 |
msg = _('"role=subject" or "role=object" must be specified in options') |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
167 |
raise ValidationError(schemacfg.eid, {rn('options', 'subject'): msg}) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
168 |
try: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
169 |
action = options.pop('action') |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
170 |
self.action_options[action](action, options, schemacfg.eid, _) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
171 |
except KeyError: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
172 |
msg = _('"action" must be specified in options; allowed values are ' |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
173 |
'%s') % ', '.join(self.action_methods) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
174 |
raise ValidationError(schemacfg.eid, {rn('options', 'subject'): msg}) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
175 |
if not checkonly: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
176 |
if role == 'subject': |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
177 |
etype = schemacfg.schema.stype.name |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
178 |
ttype = schemacfg.schema.otype.name |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
179 |
else: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
180 |
etype = schemacfg.schema.otype.name |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
181 |
ttype = schemacfg.schema.stype.name |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
182 |
etyperules = self.source.mapping.setdefault(etype, {}) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
183 |
etyperules.setdefault((rtype, role, action), []).append( |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
184 |
(ttype, options) ) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
185 |
self.source.mapping_idx[schemacfg.eid] = ( |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
186 |
etype, rtype, role, action, ttype) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
187 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
188 |
def del_schema_config(self, schemacfg, checkonly=False): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
189 |
"""deleted CWSourceSchemaConfig, modify mapping accordingly""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
190 |
etype, rtype, role, action, ttype = self.source.mapping_idx[schemacfg.eid] |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
191 |
rules = self.source.mapping[etype][(rtype, role, action)] |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
192 |
rules = [x for x in rules if not x[0] == ttype] |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
193 |
if not rules: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
194 |
del self.source.mapping[etype][(rtype, role, action)] |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
195 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
196 |
# import handling ########################################################## |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
197 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
198 |
def process(self, url, partialcommit=True): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
199 |
"""IDataFeedParser main entry point""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
200 |
# XXX suppression support according to source configuration. If set, get |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
201 |
# all cwuri of entities from this source, and compare with newly |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
202 |
# imported ones |
7001
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
203 |
error = False |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
204 |
for item, rels in self.parse(url): |
7001
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
205 |
cwuri = item['cwuri'] |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
206 |
try: |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
207 |
self.process_item(item, rels) |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
208 |
if partialcommit: |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
209 |
# commit+set_pool instead of commit(reset_pool=False) to let |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
210 |
# other a chance to get our pool |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
211 |
self._cw.commit() |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
212 |
self._cw.set_pool() |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
213 |
except ValidationError, exc: |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
214 |
if partialcommit: |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
215 |
self.source.error('Skipping %s because of validation error %s' % (cwuri, exc)) |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
216 |
self._cw.rollback() |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
217 |
self._cw.set_pool() |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
218 |
error = True |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
219 |
else: |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
220 |
raise |
c53aa19640b2
[sobjects/parsers] on validationerror, skip entity and continue processing feed
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
6995
diff
changeset
|
221 |
return error |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
222 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
223 |
def parse(self, url): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
224 |
if not url.startswith('http'): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
225 |
stream = StringIO.StringIO(url) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
226 |
else: |
6963
5774d4ba4306
[datafeed] introduce a host mapping so dev instance may transparently redirect request to another host than the actual's one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6960
diff
changeset
|
227 |
for mappedurl in HOST_MAPPING: |
5774d4ba4306
[datafeed] introduce a host mapping so dev instance may transparently redirect request to another host than the actual's one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6960
diff
changeset
|
228 |
if url.startswith(mappedurl): |
5774d4ba4306
[datafeed] introduce a host mapping so dev instance may transparently redirect request to another host than the actual's one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6960
diff
changeset
|
229 |
url = url.replace(mappedurl, HOST_MAPPING[mappedurl], 1) |
5774d4ba4306
[datafeed] introduce a host mapping so dev instance may transparently redirect request to another host than the actual's one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6960
diff
changeset
|
230 |
break |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
231 |
self.source.info('GET %s', url) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
232 |
stream = _OPENER.open(url) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
233 |
return _entity_etree(etree.parse(stream).getroot()) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
234 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
235 |
def process_one(self, url): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
236 |
# XXX assert len(root.children) == 1 |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
237 |
for item, rels in self.parse(url): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
238 |
return self.process_item(item, rels) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
239 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
240 |
def process_item(self, item, rels): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
241 |
entity = self.extid2entity(str(item.pop('cwuri')), |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
242 |
item.pop('cwtype'), |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
243 |
item=item) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
244 |
if not (self.created_during_pull(entity) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
245 |
or self.updated_during_pull(entity)): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
246 |
self.notify_updated(entity) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
247 |
item.pop('eid') |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
248 |
# XXX check modification date |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
249 |
attrs = extract_typed_attrs(entity.e_schema, item) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
250 |
entity.set_attributes(**attrs) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
251 |
for (rtype, role, action), rules in self.source.mapping.get(entity.__regid__, {}).iteritems(): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
252 |
try: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
253 |
rel = rels[role][rtype] |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
254 |
except KeyError: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
255 |
self.source.error('relation %s-%s doesn\'t seem exported in %s xml', |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
256 |
rtype, role, entity.__regid__) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
257 |
continue |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
258 |
try: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
259 |
actionmethod = self.action_methods[action] |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
260 |
except KeyError: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
261 |
raise Exception('Unknown action %s' % action) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
262 |
actionmethod(entity, rtype, role, rel, rules) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
263 |
return entity |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
264 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
265 |
def before_entity_copy(self, entity, sourceparams): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
266 |
"""IDataFeedParser callback""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
267 |
attrs = extract_typed_attrs(entity.e_schema, sourceparams['item']) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
268 |
entity.cw_edited.update(attrs) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
269 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
270 |
def related_copy(self, entity, rtype, role, value, rules): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
271 |
"""implementation of 'copy' action |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
272 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
273 |
Takes no option. |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
274 |
""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
275 |
assert not any(x[1] for x in rules), "'copy' action takes no option" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
276 |
ttypes = set([x[0] for x in rules]) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
277 |
value = [item for item in value if item['cwtype'] in ttypes] |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
278 |
eids = [] # local eids |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
279 |
if not value: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
280 |
self._clear_relation(entity, rtype, role, ttypes) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
281 |
return |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
282 |
for item in value: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
283 |
eids.append(self.process_one(self._complete_url(item)).eid) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
284 |
self._set_relation(entity, rtype, role, eids) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
285 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
286 |
def related_link(self, entity, rtype, role, value, rules): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
287 |
"""implementation of 'link' action |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
288 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
289 |
requires an options to control search of the linked entity. |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
290 |
""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
291 |
for ttype, options in rules: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
292 |
assert 'linkattr' in options, ( |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
293 |
"'link-or-create' action require a list of attributes used to " |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
294 |
"search if the entity already exists") |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
295 |
self._related_link(entity, rtype, role, ttype, value, [options['linkattr']], |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
296 |
self._log_not_found) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
297 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
298 |
def related_link_or_create(self, entity, rtype, role, value, rules): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
299 |
"""implementation of 'link-or-create' action |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
300 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
301 |
requires an options to control search of the linked entity. |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
302 |
""" |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
303 |
for ttype, options in rules: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
304 |
assert 'linkattr' in options, ( |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
305 |
"'link-or-create' action require a list of attributes used to " |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
306 |
"search if the entity already exists") |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
307 |
self._related_link(entity, rtype, role, ttype, value, [options['linkattr']], |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
308 |
self._create_not_found) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
309 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
310 |
def _log_not_found(self, entity, rtype, role, ritem, searchvalues): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
311 |
self.source.error('can find %s entity with attributes %s', |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
312 |
ritem['cwtype'], searchvalues) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
313 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
314 |
def _create_not_found(self, entity, rtype, role, ritem, searchvalues): |
6975
75a232b2e477
[test] python 2.5 specifix
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6970
diff
changeset
|
315 |
ensure_str_keys(searchvalues) # XXX necessary with python < 2.6 |
6960
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
316 |
return self._cw.create_entity(ritem['cwtype'], **searchvalues).eid |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
317 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
318 |
def _related_link(self, entity, rtype, role, ttype, value, searchattrs, |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
319 |
notfound_callback): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
320 |
eids = [] # local eids |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
321 |
for item in value: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
322 |
if item['cwtype'] != ttype: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
323 |
continue |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
324 |
if not all(attr in item for attr in searchattrs): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
325 |
# need to fetch related entity's xml |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
326 |
ritems = list(self.parse(self._complete_url(item, False))) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
327 |
assert len(ritems) == 1, 'unexpected xml' |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
328 |
ritem = ritems[0][0] # list of 2-uples |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
329 |
assert all(attr in ritem for attr in searchattrs), \ |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
330 |
'missing attribute, got %s expected keys %s' % (item, searchattrs) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
331 |
else: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
332 |
ritem = item |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
333 |
kwargs = dict((attr, ritem[attr]) for attr in searchattrs) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
334 |
rql = build_search_rql(item['cwtype'], kwargs) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
335 |
rset = self._cw.execute(rql, kwargs) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
336 |
if rset: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
337 |
assert len(rset) == 1 |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
338 |
eids.append(rset[0][0]) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
339 |
else: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
340 |
eid = notfound_callback(entity, rtype, role, ritem, kwargs) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
341 |
if eid is not None: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
342 |
eids.append(eid) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
343 |
if not eids: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
344 |
self._clear_relation(entity, rtype, role, (ttype,)) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
345 |
else: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
346 |
self._set_relation(entity, rtype, role, eids) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
347 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
348 |
def _complete_url(self, item, add_relations=True): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
349 |
itemurl = item['cwuri'] + '?vid=xml' |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
350 |
for rtype, role, _ in self.source.mapping.get(item['cwtype'], ()): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
351 |
itemurl += '&relation=%s_%s' % (rtype, role) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
352 |
return itemurl |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
353 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
354 |
def _clear_relation(self, entity, rtype, role, ttypes): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
355 |
if entity.eid not in self.stats['created']: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
356 |
if len(ttypes) > 1: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
357 |
typerestr = ', Y is IN(%s)' % ','.join(ttypes) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
358 |
else: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
359 |
typerestr = ', Y is %s' % ','.join(ttypes) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
360 |
self._cw.execute('DELETE ' + rtype_role_rql(rtype, role) + typerestr, |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
361 |
{'x': entity.eid}) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
362 |
|
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
363 |
def _set_relation(self, entity, rtype, role, eids): |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
364 |
eidstr = ','.join(str(eid) for eid in eids) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
365 |
rql = rtype_role_rql(rtype, role) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
366 |
self._cw.execute('DELETE %s, NOT Y eid IN (%s)' % (rql, eidstr), |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
367 |
{'x': entity.eid}) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
368 |
if role == 'object': |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
369 |
rql = 'SET %s, Y eid IN (%s), NOT Y %s X' % (rql, eidstr, rtype) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
370 |
else: |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
371 |
rql = 'SET %s, Y eid IN (%s), NOT X %s Y' % (rql, eidstr, rtype) |
822f2530570d
[datafeed] add parser to import cubicweb xml
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
372 |
self._cw.execute(rql, {'x': entity.eid}) |
6963
5774d4ba4306
[datafeed] introduce a host mapping so dev instance may transparently redirect request to another host than the actual's one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6960
diff
changeset
|
373 |
|
5774d4ba4306
[datafeed] introduce a host mapping so dev instance may transparently redirect request to another host than the actual's one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6960
diff
changeset
|
374 |
def registration_callback(vreg): |
6970
a6ccbfbacf3d
[parser host mapping] take care to cases where apphome is None (c-c i18ncube command for instance)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6963
diff
changeset
|
375 |
vreg.register_all(globals().values(), __name__) |
6963
5774d4ba4306
[datafeed] introduce a host mapping so dev instance may transparently redirect request to another host than the actual's one
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6960
diff
changeset
|
376 |
global HOST_MAPPING |
6970
a6ccbfbacf3d
[parser host mapping] take care to cases where apphome is None (c-c i18ncube command for instance)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6963
diff
changeset
|
377 |
HOST_MAPPING = {} |
a6ccbfbacf3d
[parser host mapping] take care to cases where apphome is None (c-c i18ncube command for instance)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6963
diff
changeset
|
378 |
if vreg.config.apphome: |
a6ccbfbacf3d
[parser host mapping] take care to cases where apphome is None (c-c i18ncube command for instance)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6963
diff
changeset
|
379 |
host_mapping_file = osp.join(vreg.config.apphome, 'hostmapping.py') |
a6ccbfbacf3d
[parser host mapping] take care to cases where apphome is None (c-c i18ncube command for instance)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6963
diff
changeset
|
380 |
if osp.exists(host_mapping_file): |
a6ccbfbacf3d
[parser host mapping] take care to cases where apphome is None (c-c i18ncube command for instance)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6963
diff
changeset
|
381 |
HOST_MAPPING = eval(file(host_mapping_file).read()) |
a6ccbfbacf3d
[parser host mapping] take care to cases where apphome is None (c-c i18ncube command for instance)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6963
diff
changeset
|
382 |
vreg.info('using host mapping %s from %s', HOST_MAPPING, host_mapping_file) |