server/test/unittest_datafeed.py
author Julien Cristau <julien.cristau@logilab.fr>
Thu, 08 Jan 2015 12:22:43 +0100
changeset 10146 78f541f50369
parent 10089 6346f53c85f1
child 10774 0361442e2633
permissions -rw-r--r--
[doc] remove nonexistent css from theme
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
9798
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
     1
# copyright 2011-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     2
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     3
#
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     4
# This file is part of CubicWeb.
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     5
#
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     6
# CubicWeb is free software: you can redistribute it and/or modify it under the
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     7
# terms of the GNU Lesser General Public License as published by the Free
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     8
# Software Foundation, either version 2.1 of the License, or (at your option)
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     9
# any later version.
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    10
#
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    12
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    13
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    14
# details.
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    15
#
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    16
# You should have received a copy of the GNU Lesser General Public License along
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    17
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    18
9824
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
    19
import mimetools
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    20
from datetime import timedelta
9824
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
    21
from contextlib import contextmanager
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    22
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    23
from cubicweb.devtools.testlib import CubicWebTC
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    24
from cubicweb.server.sources import datafeed
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    25
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    26
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    27
class DataFeedTC(CubicWebTC):
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    28
    def setup_database(self):
9798
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    29
        with self.admin_access.repo_cnx() as cnx:
9824
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
    30
            with self.base_parser(cnx):
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
    31
                cnx.create_entity('CWSource', name=u'myfeed', type=u'datafeed',
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
    32
                                  parser=u'testparser', url=u'ignored',
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
    33
                                  config=u'synchronization-interval=1min')
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
    34
                cnx.commit()
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    35
9824
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
    36
    @contextmanager
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
    37
    def base_parser(self, session):
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    38
        class AParser(datafeed.DataFeedParser):
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    39
            __regid__ = 'testparser'
7386
206890413858 [repo test] datafeed parser's process method should now take raise_on_error argument
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6957
diff changeset
    40
            def process(self, url, raise_on_error=False):
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    41
                entity = self.extid2entity('http://www.cubicweb.org/', 'Card',
7978
b6150210f692 [test] fix datafeed test, we have to use an internal session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7735
diff changeset
    42
                                           item={'title': u'cubicweb.org',
10089
6346f53c85f1 [datafeed] Add a raise_on_error parameter to DataFeedSource.extid2entity
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 9897
diff changeset
    43
                                                 'content': u'the cw web site'},
6346f53c85f1 [datafeed] Add a raise_on_error parameter to DataFeedSource.extid2entity
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 9897
diff changeset
    44
                                           raise_on_error=raise_on_error)
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    45
                if not self.created_during_pull(entity):
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    46
                    self.notify_updated(entity)
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    47
            def before_entity_copy(self, entity, sourceparams):
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    48
                entity.cw_edited.update(sourceparams['item'])
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    49
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    50
        with self.temporary_appobjects(AParser):
9824
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
    51
            if 'myfeed' in self.repo.sources_by_uri:
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
    52
                yield self.repo.sources_by_uri['myfeed']._get_parser(session)
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
    53
            else:
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
    54
                yield
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    55
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    56
    def test(self):
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    57
        self.assertIn('myfeed', self.repo.sources_by_uri)
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    58
        dfsource = self.repo.sources_by_uri['myfeed']
9822
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
    59
        self.assertNotIn('use_cwuri_as_url', dfsource.__dict__)
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
    60
        self.assertEqual({'type': u'datafeed', 'uri': u'myfeed', 'use-cwuri-as-url': True},
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
    61
                         dfsource.public_config)
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
    62
        self.assertEqual(dfsource.use_cwuri_as_url, True)
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    63
        self.assertEqual(dfsource.latest_retrieval, None)
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    64
        self.assertEqual(dfsource.synchro_interval, timedelta(seconds=60))
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    65
        self.assertFalse(dfsource.fresh())
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    66
9824
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
    67
        with self.repo.internal_cnx() as cnx:
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
    68
            with self.base_parser(cnx):
9798
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    69
                stats = dfsource.pull_data(cnx, force=True)
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    70
                cnx.commit()
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    71
                # test import stats
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    72
                self.assertEqual(sorted(stats), ['checked', 'created', 'updated'])
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    73
                self.assertEqual(len(stats['created']), 1)
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    74
                entity = cnx.execute('Card X').get_entity(0, 0)
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    75
                self.assertIn(entity.eid, stats['created'])
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    76
                self.assertEqual(stats['updated'], set())
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    77
                # test imported entities
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    78
                self.assertEqual(entity.title, 'cubicweb.org')
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    79
                self.assertEqual(entity.content, 'the cw web site')
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    80
                self.assertEqual(entity.cwuri, 'http://www.cubicweb.org/')
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    81
                self.assertEqual(entity.cw_source[0].name, 'myfeed')
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    82
                self.assertEqual(entity.cw_metainformation(),
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    83
                                 {'type': 'Card',
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    84
                                  'source': {'uri': 'myfeed', 'type': 'datafeed', 'use-cwuri-as-url': True},
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    85
                                  'extid': 'http://www.cubicweb.org/'}
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    86
                                 )
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    87
                self.assertEqual(entity.absolute_url(), 'http://www.cubicweb.org/')
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    88
                # test repo cache keys
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    89
                self.assertEqual(self.repo._type_source_cache[entity.eid],
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    90
                                 ('Card', 'http://www.cubicweb.org/', 'myfeed'))
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    91
                self.assertEqual(self.repo._extid_cache['http://www.cubicweb.org/'],
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    92
                                 entity.eid)
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    93
                # test repull
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    94
                stats = dfsource.pull_data(cnx, force=True)
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    95
                self.assertEqual(stats['created'], set())
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    96
                self.assertEqual(stats['updated'], set((entity.eid,)))
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    97
                # test repull with caches reseted
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    98
                self.repo._type_source_cache.clear()
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
    99
                self.repo._extid_cache.clear()
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   100
                stats = dfsource.pull_data(cnx, force=True)
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   101
                self.assertEqual(stats['created'], set())
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   102
                self.assertEqual(stats['updated'], set((entity.eid,)))
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   103
                self.assertEqual(self.repo._type_source_cache[entity.eid],
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   104
                                 ('Card', 'http://www.cubicweb.org/', 'myfeed'))
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   105
                self.assertEqual(self.repo._extid_cache['http://www.cubicweb.org/'],
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   106
                                 entity.eid)
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   107
9860
e24bf60428d3 [tests/datafeed] use the new connection api (a small leftover)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9798
diff changeset
   108
                self.assertEqual(dfsource.source_cwuris(cnx),
e24bf60428d3 [tests/datafeed] use the new connection api (a small leftover)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9798
diff changeset
   109
                                 {'http://www.cubicweb.org/': (entity.eid, 'Card')}
e24bf60428d3 [tests/datafeed] use the new connection api (a small leftover)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9798
diff changeset
   110
                             )
e24bf60428d3 [tests/datafeed] use the new connection api (a small leftover)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9798
diff changeset
   111
                self.assertTrue(dfsource.latest_retrieval)
e24bf60428d3 [tests/datafeed] use the new connection api (a small leftover)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9798
diff changeset
   112
                self.assertTrue(dfsource.fresh())
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   113
7735
71adfd6bab38 [source] update system table and internal structures on source renaming. Closes #1896721
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7717
diff changeset
   114
        # test_rename_source
9798
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   115
        with self.admin_access.repo_cnx() as cnx:
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   116
            cnx.execute('SET S name "myrenamedfeed" WHERE S is CWSource, S name "myfeed"')
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   117
            cnx.commit()
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   118
            entity = cnx.execute('Card X').get_entity(0, 0)
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   119
            self.assertEqual(entity.cwuri, 'http://www.cubicweb.org/')
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   120
            self.assertEqual(entity.cw_source[0].name, 'myrenamedfeed')
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   121
            self.assertEqual(entity.cw_metainformation(),
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   122
                             {'type': 'Card',
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   123
                              'source': {'uri': 'myrenamedfeed', 'type': 'datafeed', 'use-cwuri-as-url': True},
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   124
                              'extid': 'http://www.cubicweb.org/'}
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   125
                             )
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   126
            self.assertEqual(self.repo._type_source_cache[entity.eid],
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   127
                             ('Card', 'http://www.cubicweb.org/', 'myrenamedfeed'))
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   128
            self.assertEqual(self.repo._extid_cache['http://www.cubicweb.org/'],
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   129
                             entity.eid)
7735
71adfd6bab38 [source] update system table and internal structures on source renaming. Closes #1896721
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7717
diff changeset
   130
9798
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   131
            # test_delete_source
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   132
            cnx.execute('DELETE CWSource S WHERE S name "myrenamedfeed"')
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   133
            cnx.commit()
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   134
            self.assertFalse(cnx.execute('Card X WHERE X title "cubicweb.org"'))
1c105cf42904 [tests/datafeed] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9469
diff changeset
   135
            self.assertFalse(cnx.execute('Any X WHERE X has_text "cubicweb.org"'))
7650
278fe9c1f3ad [repo] closes #1821172: fix source deletion
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7631
diff changeset
   136
9824
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
   137
    def test_parser_retrieve_url_local(self):
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
   138
        with self.admin_access.repo_cnx() as cnx:
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
   139
            with self.base_parser(cnx) as parser:
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
   140
                value = parser.retrieve_url('a string')
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
   141
                self.assertEqual(200, value.getcode())
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
   142
                self.assertEqual('a string', value.geturl())
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
   143
                self.assertIsInstance(value.info(), mimetools.Message)
30183ecf5c61 [datafeed parser] fix retrieve_url to always return urllib2.urlopen compatible output
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9822
diff changeset
   144
9822
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
   145
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
   146
class DataFeedConfigTC(CubicWebTC):
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
   147
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
   148
    def test_use_cwuri_as_url_override(self):
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
   149
        with self.admin_access.client_cnx() as cnx:
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
   150
            cnx.create_entity('CWSource', name=u'myfeed', type=u'datafeed',
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
   151
                              parser=u'testparser', url=u'ignored',
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
   152
                              config=u'use-cwuri-as-url=no')
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
   153
            cnx.commit()
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
   154
        dfsource = self.repo.sources_by_uri['myfeed']
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
   155
        self.assertEqual(dfsource.use_cwuri_as_url, False)
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
   156
        self.assertEqual({'type': u'datafeed', 'uri': u'myfeed', 'use-cwuri-as-url': False},
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
   157
                         dfsource.public_config)
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9798
diff changeset
   158
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   159
if __name__ == '__main__':
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   160
    from logilab.common.testlib import unittest_main
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   161
    unittest_main()