author | Sylvain Thénault <sylvain.thenault@logilab.fr> |
Fri, 30 Sep 2016 17:18:15 +0200 | |
changeset 11759 | 7b7108eb8178 |
parent 11758 | 3f81636a75db |
child 11762 | 7518cb58ab4c |
permissions | -rw-r--r-- |
10832
2350424585f8
[server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10805
diff
changeset
|
1 |
# coding: utf-8 |
11758
3f81636a75db
[datafeed] Drop entity deletion handling in the default source / parser
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11756
diff
changeset
|
2 |
# copyright 2011-2016 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
3 |
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
4 |
# |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
5 |
# This file is part of CubicWeb. |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
6 |
# |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
7 |
# CubicWeb is free software: you can redistribute it and/or modify it under the |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
8 |
# terms of the GNU Lesser General Public License as published by the Free |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
9 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
10 |
# any later version. |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
11 |
# |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
12 |
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
13 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
14 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
15 |
# details. |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
16 |
# |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
17 |
# You should have received a copy of the GNU Lesser General Public License along |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
18 |
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
19 |
|
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
20 |
from datetime import timedelta |
9824
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
21 |
from contextlib import contextmanager |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
22 |
|
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
23 |
from cubicweb.devtools.testlib import CubicWebTC |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
24 |
from cubicweb.server.sources import datafeed |
11756
60fed6272771
[repository] Drop deprecated extid2eid API and friends
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11740
diff
changeset
|
25 |
from cubicweb.dataimport.stores import NoHookRQLObjectStore, MetaGenerator |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
26 |
|
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
27 |
|
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
28 |
class DataFeedTC(CubicWebTC): |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
29 |
def setup_database(self): |
9798
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
30 |
with self.admin_access.repo_cnx() as cnx: |
9824
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
31 |
with self.base_parser(cnx): |
10832
2350424585f8
[server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10805
diff
changeset
|
32 |
cnx.create_entity('CWSource', name=u'ô myfeed', type=u'datafeed', |
9824
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
33 |
parser=u'testparser', url=u'ignored', |
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
34 |
config=u'synchronization-interval=1min') |
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
35 |
cnx.commit() |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
36 |
|
9824
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
37 |
@contextmanager |
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
38 |
def base_parser(self, session): |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
39 |
class AParser(datafeed.DataFeedParser): |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
40 |
__regid__ = 'testparser' |
11756
60fed6272771
[repository] Drop deprecated extid2eid API and friends
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11740
diff
changeset
|
41 |
|
7386
206890413858
[repo test] datafeed parser's process method should now take raise_on_error argument
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6957
diff
changeset
|
42 |
def process(self, url, raise_on_error=False): |
11756
60fed6272771
[repository] Drop deprecated extid2eid API and friends
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11740
diff
changeset
|
43 |
metagenerator = MetaGenerator(self._cw, source=self.source) |
60fed6272771
[repository] Drop deprecated extid2eid API and friends
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11740
diff
changeset
|
44 |
store = NoHookRQLObjectStore(self._cw, metagenerator) |
60fed6272771
[repository] Drop deprecated extid2eid API and friends
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11740
diff
changeset
|
45 |
store.prepare_insert_entity('Card', |
60fed6272771
[repository] Drop deprecated extid2eid API and friends
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11740
diff
changeset
|
46 |
cwuri=u'http://www.cubicweb.org/', |
60fed6272771
[repository] Drop deprecated extid2eid API and friends
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11740
diff
changeset
|
47 |
title=u'cubicweb.org', |
60fed6272771
[repository] Drop deprecated extid2eid API and friends
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11740
diff
changeset
|
48 |
content=u'the cw web site') |
60fed6272771
[repository] Drop deprecated extid2eid API and friends
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11740
diff
changeset
|
49 |
store.flush() |
60fed6272771
[repository] Drop deprecated extid2eid API and friends
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11740
diff
changeset
|
50 |
store.commit() |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
51 |
|
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
52 |
with self.temporary_appobjects(AParser): |
10832
2350424585f8
[server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10805
diff
changeset
|
53 |
if u'ô myfeed' in self.repo.sources_by_uri: |
2350424585f8
[server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10805
diff
changeset
|
54 |
yield self.repo.sources_by_uri[u'ô myfeed']._get_parser(session) |
9824
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
55 |
else: |
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
56 |
yield |
11740
dabbb2a4a493
[datafeed] Complete the import log even if parser could not be found
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
11252
diff
changeset
|
57 |
# vreg.unregister just pops appobjects from their regid entry, |
dabbb2a4a493
[datafeed] Complete the import log even if parser could not be found
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
11252
diff
changeset
|
58 |
# completely remove the entry to ensure we have no side effect with |
dabbb2a4a493
[datafeed] Complete the import log even if parser could not be found
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
11252
diff
changeset
|
59 |
# this empty entry. |
dabbb2a4a493
[datafeed] Complete the import log even if parser could not be found
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
11252
diff
changeset
|
60 |
del self.vreg['parsers'][AParser.__regid__] |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
61 |
|
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
62 |
def test(self): |
10832
2350424585f8
[server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10805
diff
changeset
|
63 |
self.assertIn(u'ô myfeed', self.repo.sources_by_uri) |
2350424585f8
[server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10805
diff
changeset
|
64 |
dfsource = self.repo.sources_by_uri[u'ô myfeed'] |
9822
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
65 |
self.assertNotIn('use_cwuri_as_url', dfsource.__dict__) |
10832
2350424585f8
[server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10805
diff
changeset
|
66 |
self.assertEqual({'type': u'datafeed', 'uri': u'ô myfeed', 'use-cwuri-as-url': True}, |
9822
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
67 |
dfsource.public_config) |
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
68 |
self.assertEqual(dfsource.use_cwuri_as_url, True) |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
69 |
self.assertEqual(dfsource.latest_retrieval, None) |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
70 |
self.assertEqual(dfsource.synchro_interval, timedelta(seconds=60)) |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
71 |
self.assertFalse(dfsource.fresh()) |
10832
2350424585f8
[server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10805
diff
changeset
|
72 |
# ensure source's logger name has been unormalized |
2350424585f8
[server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10805
diff
changeset
|
73 |
self.assertEqual(dfsource.info.__self__.name, 'cubicweb.sources.o myfeed') |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
74 |
|
9824
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
75 |
with self.repo.internal_cnx() as cnx: |
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
76 |
with self.base_parser(cnx): |
11756
60fed6272771
[repository] Drop deprecated extid2eid API and friends
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
11740
diff
changeset
|
77 |
stats = dfsource.pull_data(cnx, force=True, raise_on_error=True) |
9798
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
78 |
cnx.commit() |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
79 |
# test import stats |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
80 |
self.assertEqual(sorted(stats), ['checked', 'created', 'updated']) |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
81 |
entity = cnx.execute('Card X').get_entity(0, 0) |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
82 |
# test imported entities |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
83 |
self.assertEqual(entity.title, 'cubicweb.org') |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
84 |
self.assertEqual(entity.content, 'the cw web site') |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
85 |
self.assertEqual(entity.cwuri, 'http://www.cubicweb.org/') |
10832
2350424585f8
[server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10805
diff
changeset
|
86 |
self.assertEqual(entity.cw_source[0].name, u'ô myfeed') |
9798
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
87 |
self.assertEqual(entity.cw_metainformation(), |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
88 |
{'type': 'Card', |
10832
2350424585f8
[server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10805
diff
changeset
|
89 |
'source': {'uri': u'ô myfeed', 'type': 'datafeed', 'use-cwuri-as-url': True}, |
10774
0361442e2633
[server/test] use bytes for extid
Julien Cristau <julien.cristau@logilab.fr>
parents:
10089
diff
changeset
|
90 |
'extid': b'http://www.cubicweb.org/'} |
9798
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
91 |
) |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
92 |
# test repo cache keys |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
93 |
self.assertEqual(self.repo._type_source_cache[entity.eid], |
10832
2350424585f8
[server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10805
diff
changeset
|
94 |
('Card', b'http://www.cubicweb.org/', u'ô myfeed')) |
9798
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
95 |
self.assertEqual(self.repo._type_source_cache[entity.eid], |
10832
2350424585f8
[server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10805
diff
changeset
|
96 |
('Card', b'http://www.cubicweb.org/', u'ô myfeed')) |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
97 |
|
9860
e24bf60428d3
[tests/datafeed] use the new connection api (a small leftover)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9798
diff
changeset
|
98 |
self.assertTrue(dfsource.latest_retrieval) |
e24bf60428d3
[tests/datafeed] use the new connection api (a small leftover)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9798
diff
changeset
|
99 |
self.assertTrue(dfsource.fresh()) |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
100 |
|
7735
71adfd6bab38
[source] update system table and internal structures on source renaming. Closes #1896721
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7717
diff
changeset
|
101 |
# test_rename_source |
9798
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
102 |
with self.admin_access.repo_cnx() as cnx: |
10832
2350424585f8
[server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10805
diff
changeset
|
103 |
cnx.entity_from_eid(dfsource.eid).cw_set(name=u"myrenamedfeed") |
9798
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
104 |
cnx.commit() |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
105 |
entity = cnx.execute('Card X').get_entity(0, 0) |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
106 |
self.assertEqual(entity.cwuri, 'http://www.cubicweb.org/') |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
107 |
self.assertEqual(entity.cw_source[0].name, 'myrenamedfeed') |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
108 |
self.assertEqual(entity.cw_metainformation(), |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
109 |
{'type': 'Card', |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
110 |
'source': {'uri': 'myrenamedfeed', 'type': 'datafeed', 'use-cwuri-as-url': True}, |
10774
0361442e2633
[server/test] use bytes for extid
Julien Cristau <julien.cristau@logilab.fr>
parents:
10089
diff
changeset
|
111 |
'extid': b'http://www.cubicweb.org/'} |
9798
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
112 |
) |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
113 |
self.assertEqual(self.repo._type_source_cache[entity.eid], |
10774
0361442e2633
[server/test] use bytes for extid
Julien Cristau <julien.cristau@logilab.fr>
parents:
10089
diff
changeset
|
114 |
('Card', b'http://www.cubicweb.org/', 'myrenamedfeed')) |
7735
71adfd6bab38
[source] update system table and internal structures on source renaming. Closes #1896721
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7717
diff
changeset
|
115 |
|
9798
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
116 |
# test_delete_source |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
117 |
cnx.execute('DELETE CWSource S WHERE S name "myrenamedfeed"') |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
118 |
cnx.commit() |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
119 |
self.assertFalse(cnx.execute('Card X WHERE X title "cubicweb.org"')) |
1c105cf42904
[tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9469
diff
changeset
|
120 |
self.assertFalse(cnx.execute('Any X WHERE X has_text "cubicweb.org"')) |
7650
278fe9c1f3ad
[repo] closes #1821172: fix source deletion
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7631
diff
changeset
|
121 |
|
9824
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
122 |
def test_parser_retrieve_url_local(self): |
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
123 |
with self.admin_access.repo_cnx() as cnx: |
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
124 |
with self.base_parser(cnx) as parser: |
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
125 |
value = parser.retrieve_url('a string') |
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
126 |
self.assertEqual(200, value.getcode()) |
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
127 |
self.assertEqual('a string', value.geturl()) |
30183ecf5c61
[datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9822
diff
changeset
|
128 |
|
10956
208c9ac8edbb
update repo sources on update of source's entities .url attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10832
diff
changeset
|
129 |
def test_update_url(self): |
208c9ac8edbb
update repo sources on update of source's entities .url attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10832
diff
changeset
|
130 |
dfsource = self.repo.sources_by_uri[u'ô myfeed'] |
208c9ac8edbb
update repo sources on update of source's entities .url attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10832
diff
changeset
|
131 |
with self.admin_access.repo_cnx() as cnx: |
208c9ac8edbb
update repo sources on update of source's entities .url attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10832
diff
changeset
|
132 |
cnx.entity_from_eid(dfsource.eid).cw_set(url=u"http://pouet.com\nhttp://pouet.org") |
208c9ac8edbb
update repo sources on update of source's entities .url attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10832
diff
changeset
|
133 |
self.assertEqual(dfsource.urls, [u'ignored']) |
208c9ac8edbb
update repo sources on update of source's entities .url attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10832
diff
changeset
|
134 |
cnx.commit() |
208c9ac8edbb
update repo sources on update of source's entities .url attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10832
diff
changeset
|
135 |
self.assertEqual(dfsource.urls, [u"http://pouet.com", u"http://pouet.org"]) |
208c9ac8edbb
update repo sources on update of source's entities .url attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10832
diff
changeset
|
136 |
|
11740
dabbb2a4a493
[datafeed] Complete the import log even if parser could not be found
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
11252
diff
changeset
|
137 |
def test_parser_not_found(self): |
dabbb2a4a493
[datafeed] Complete the import log even if parser could not be found
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
11252
diff
changeset
|
138 |
dfsource = self.repo.sources_by_uri[u'ô myfeed'] |
dabbb2a4a493
[datafeed] Complete the import log even if parser could not be found
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
11252
diff
changeset
|
139 |
with self.assertLogs('cubicweb.sources.o myfeed', level='ERROR') as cm: |
dabbb2a4a493
[datafeed] Complete the import log even if parser could not be found
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
11252
diff
changeset
|
140 |
with self.repo.internal_cnx() as cnx: |
dabbb2a4a493
[datafeed] Complete the import log even if parser could not be found
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
11252
diff
changeset
|
141 |
stats = dfsource.pull_data(cnx, force=True) |
dabbb2a4a493
[datafeed] Complete the import log even if parser could not be found
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
11252
diff
changeset
|
142 |
importlog = cnx.find('CWDataImport').one().log |
dabbb2a4a493
[datafeed] Complete the import log even if parser could not be found
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
11252
diff
changeset
|
143 |
self.assertIn('failed to load parser for', cm.output[0]) |
dabbb2a4a493
[datafeed] Complete the import log even if parser could not be found
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
11252
diff
changeset
|
144 |
self.assertEqual(stats, {}) |
dabbb2a4a493
[datafeed] Complete the import log even if parser could not be found
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
11252
diff
changeset
|
145 |
self.assertIn(u'failed to load parser for source "ô myfeed"', |
dabbb2a4a493
[datafeed] Complete the import log even if parser could not be found
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
11252
diff
changeset
|
146 |
importlog) |
dabbb2a4a493
[datafeed] Complete the import log even if parser could not be found
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
11252
diff
changeset
|
147 |
|
9822
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
148 |
|
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
149 |
class DataFeedConfigTC(CubicWebTC): |
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
150 |
|
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
151 |
def test_use_cwuri_as_url_override(self): |
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
152 |
with self.admin_access.client_cnx() as cnx: |
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
153 |
cnx.create_entity('CWSource', name=u'myfeed', type=u'datafeed', |
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
154 |
parser=u'testparser', url=u'ignored', |
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
155 |
config=u'use-cwuri-as-url=no') |
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
156 |
cnx.commit() |
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
157 |
dfsource = self.repo.sources_by_uri['myfeed'] |
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
158 |
self.assertEqual(dfsource.use_cwuri_as_url, False) |
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
159 |
self.assertEqual({'type': u'datafeed', 'uri': u'myfeed', 'use-cwuri-as-url': False}, |
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
160 |
dfsource.public_config) |
4a118bfd6ab4
[datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9798
diff
changeset
|
161 |
|
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
162 |
if __name__ == '__main__': |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
163 |
from logilab.common.testlib import unittest_main |
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
164 |
unittest_main() |