# HG changeset patch # User Denis Laxalde # Date 1491397149 -7200 # Node ID d540defa0591692254e2d210d8757145dfc6e137 # Parent 752b94ed974831be924b2f52e0b26af9493787d2 [server] Add source_by_eid and source_by_uri methods to repository Most of the times we only need to retrieve one source (either by uri or eid) and querying sources_by_eid and sources_by_uri properties on repository just for one item is costly. So these methods query what's needed. We issue a ValueError (instead of KeyError for sources_by_{eid,uri} dict) in case the key is not found. diff -r 752b94ed9748 -r d540defa0591 cubicweb/dataimport/test/test_massive_store.py --- a/cubicweb/dataimport/test/test_massive_store.py Tue Apr 04 17:43:56 2017 +0200 +++ b/cubicweb/dataimport/test/test_massive_store.py Wed Apr 05 14:59:09 2017 +0200 @@ -45,7 +45,7 @@ def store_impl(self, cnx): source = cnx.create_entity('CWSource', type=u'datafeed', name=u'test', url=u'test') cnx.commit() - metagen = stores.MetadataGenerator(cnx, source=cnx.repo.sources_by_eid[source.eid]) + metagen = stores.MetadataGenerator(cnx, source=cnx.repo.source_by_eid(source.eid)) return MassiveObjectStore(cnx, metagen=metagen) diff -r 752b94ed9748 -r d540defa0591 cubicweb/dataimport/test/test_stores.py --- a/cubicweb/dataimport/test/test_stores.py Tue Apr 04 17:43:56 2017 +0200 +++ b/cubicweb/dataimport/test/test_stores.py Wed Apr 05 14:59:09 2017 +0200 @@ -80,7 +80,7 @@ def store_impl(self, cnx): source = cnx.create_entity('CWSource', type=u'datafeed', name=u'test', url=u'test') cnx.commit() - metagen = stores.MetadataGenerator(cnx, source=cnx.repo.sources_by_eid[source.eid]) + metagen = stores.MetadataGenerator(cnx, source=cnx.repo.source_by_eid(source.eid)) return stores.NoHookRQLObjectStore(cnx, metagen) diff -r 752b94ed9748 -r d540defa0591 cubicweb/entities/sources.py --- a/cubicweb/entities/sources.py Tue Apr 04 17:43:56 2017 +0200 +++ b/cubicweb/entities/sources.py Wed Apr 05 14:59:09 2017 +0200 @@ -63,7 +63,7 @@ """repository only property, not available from the web side (eg self._cw is expected to be a server session) """ - return self._cw.repo.sources_by_eid[self.eid] + return self._cw.repo.source_by_eid(self.eid) class CWSourceHostConfig(_CWSourceCfgMixIn, AnyEntity): diff -r 752b94ed9748 -r d540defa0591 cubicweb/misc/scripts/ldap_change_base_dn.py --- a/cubicweb/misc/scripts/ldap_change_base_dn.py Tue Apr 04 17:43:56 2017 +0200 +++ b/cubicweb/misc/scripts/ldap_change_base_dn.py Wed Apr 05 14:59:09 2017 +0200 @@ -8,7 +8,7 @@ print() print('you should not have updated your sources file yet') -olddn = repo.sources_by_uri[uri].config['user-base-dn'] +olddn = repo.source_by_uri(uri).config['user-base-dn'] assert olddn != newdn diff -r 752b94ed9748 -r d540defa0591 cubicweb/server/repository.py --- a/cubicweb/server/repository.py Tue Apr 04 17:43:56 2017 +0200 +++ b/cubicweb/server/repository.py Wed Apr 05 14:59:09 2017 +0200 @@ -302,6 +302,20 @@ # 5. call instance level initialisation hooks self.hm.call_hooks('server_startup', repo=self) + def source_by_uri(self, uri): + with self.internal_cnx() as cnx: + rset = cnx.find('CWSource', name=uri) + if not rset: + raise ValueError('no source with uri %s found' % uri) + return self._source_from_cwsource(rset.one()) + + def source_by_eid(self, eid): + with self.internal_cnx() as cnx: + rset = cnx.find('CWSource', eid=eid) + if not rset: + raise ValueError('no source with eid %d found' % eid) + return self._source_from_cwsource(rset.one()) + @property def sources_by_uri(self): mapping = {'system': self.system_source} @@ -323,20 +337,24 @@ for sourceent in cnx.execute( 'Any S, SN, SA, SC WHERE S is_instance_of CWSource, ' 'S name SN, S type SA, S config SC, S name != "system"').entities(): - source = self.get_source(sourceent.type, sourceent.name, - sourceent.host_config, sourceent.eid) - if self.config.source_enabled(source): - # call source's init method to complete their initialisation if - # needed (for instance looking for persistent configuration using an - # internal session, which is not possible until connections sets have been - # initialized) - source.init(True, sourceent) - else: - source.init(False, sourceent) - source.set_schema(self.schema) + source = self._source_from_cwsource(sourceent) yield sourceent, source self._clear_source_defs_caches() + def _source_from_cwsource(self, sourceent): + source = self.get_source(sourceent.type, sourceent.name, + sourceent.host_config, sourceent.eid) + if self.config.source_enabled(source): + # call source's init method to complete their initialisation if + # needed (for instance looking for persistent configuration using an + # internal session, which is not possible until connections sets have been + # initialized) + source.init(True, sourceent) + else: + source.init(False, sourceent) + source.set_schema(self.schema) + return source + # internals ############################################################### def _init_system_source(self): diff -r 752b94ed9748 -r d540defa0591 cubicweb/server/serverctl.py --- a/cubicweb/server/serverctl.py Tue Apr 04 17:43:56 2017 +0200 +++ b/cubicweb/server/serverctl.py Wed Apr 05 14:59:09 2017 +0200 @@ -1056,8 +1056,8 @@ if len(args) >= 2: for name in args[1:]: try: - source = repo.sources_by_uri[name] - except KeyError: + source = repo.source_by_uri(name) + except ValueError: cnx.error('no source named %r' % name) errors = True else: diff -r 752b94ed9748 -r d540defa0591 cubicweb/server/sources/datafeed.py --- a/cubicweb/server/sources/datafeed.py Tue Apr 04 17:43:56 2017 +0200 +++ b/cubicweb/server/sources/datafeed.py Wed Apr 05 14:59:09 2017 +0200 @@ -191,7 +191,7 @@ def _synchronize_source(repo, source_eid, import_log_eid): with repo.internal_cnx() as cnx: - source = repo.sources_by_eid[source_eid] + source = repo.source_by_eid(source_eid) source._pull_data(cnx, force, raise_on_error, import_log_eid=import_log_eid) sync = partial(_synchronize_source, cnx.repo, self.eid, import_log.eid) diff -r 752b94ed9748 -r d540defa0591 cubicweb/server/test/unittest_datafeed.py --- a/cubicweb/server/test/unittest_datafeed.py Tue Apr 04 17:43:56 2017 +0200 +++ b/cubicweb/server/test/unittest_datafeed.py Wed Apr 05 14:59:09 2017 +0200 @@ -50,18 +50,19 @@ store.commit() with self.temporary_appobjects(AParser): - if u'ô myfeed' in self.repo.sources_by_uri: - yield self.repo.sources_by_uri[u'ô myfeed']._get_parser(session) + try: + source = self.repo.source_by_uri(u'ô myfeed') + except ValueError: + yield else: - yield + yield source._get_parser(session) # vreg.unregister just pops appobjects from their regid entry, # completely remove the entry to ensure we have no side effect with # this empty entry. del self.vreg['parsers'][AParser.__regid__] def test(self): - self.assertIn(u'ô myfeed', self.repo.sources_by_uri) - dfsource = self.repo.sources_by_uri[u'ô myfeed'] + dfsource = self.repo.source_by_uri(u'ô myfeed') self.assertNotIn('use_cwuri_as_url', dfsource.__dict__) self.assertEqual({'type': u'datafeed', 'uri': u'ô myfeed', 'use-cwuri-as-url': True}, dfsource.public_config) @@ -113,16 +114,16 @@ self.assertEqual('a string', value.geturl()) def test_update_url(self): - dfsource = self.repo.sources_by_uri[u'ô myfeed'] + dfsource = self.repo.source_by_uri(u'ô myfeed') with self.admin_access.repo_cnx() as cnx: cnx.entity_from_eid(dfsource.eid).cw_set(url=u"http://pouet.com\nhttp://pouet.org") cnx.commit() self.assertEqual(dfsource.urls, [u'ignored']) - dfsource = self.repo.sources_by_uri[u'ô myfeed'] + dfsource = self.repo.source_by_uri(u'ô myfeed') self.assertEqual(dfsource.urls, [u"http://pouet.com", u"http://pouet.org"]) def test_parser_not_found(self): - dfsource = self.repo.sources_by_uri[u'ô myfeed'] + dfsource = self.repo.source_by_uri(u'ô myfeed') with self.assertLogs('cubicweb.sources.o myfeed', level='ERROR') as cm: with self.repo.internal_cnx() as cnx: stats = dfsource.pull_data(cnx, force=True) @@ -141,7 +142,7 @@ parser=u'testparser', url=u'ignored', config=u'use-cwuri-as-url=no') cnx.commit() - dfsource = self.repo.sources_by_uri['myfeed'] + dfsource = self.repo.source_by_uri('myfeed') self.assertEqual(dfsource.use_cwuri_as_url, False) self.assertEqual({'type': u'datafeed', 'uri': u'myfeed', 'use-cwuri-as-url': False}, dfsource.public_config) diff -r 752b94ed9748 -r d540defa0591 cubicweb/server/test/unittest_ldapsource.py --- a/cubicweb/server/test/unittest_ldapsource.py Tue Apr 04 17:43:56 2017 +0200 +++ b/cubicweb/server/test/unittest_ldapsource.py Wed Apr 05 14:59:09 2017 +0200 @@ -159,7 +159,7 @@ @staticmethod def pull(cnx): - lfsource = cnx.repo.sources_by_uri['ldap'] + lfsource = cnx.repo.source_by_uri('ldap') stats = lfsource.pull_data(cnx, force=True, raise_on_error=True) cnx.commit() return stats @@ -208,7 +208,7 @@ self._ldapmodify(modcmd) def _ldapmodify(self, modcmd): - uri = self.repo.sources_by_uri['ldap'].urls[0] + uri = self.repo.source_by_uri('ldap').urls[0] updatecmd = ['ldapmodify', '-H', uri, '-v', '-x', '-D', 'cn=admin,dc=cubicweb,dc=test', '-w', 'cw'] PIPE = subprocess.PIPE @@ -247,7 +247,7 @@ self.assertTrue(entity.modification_date) def test_authenticate(self): - source = self.repo.sources_by_uri['ldap'] + source = self.repo.source_by_uri('ldap') with self.admin_access.repo_cnx() as cnx: # ensure we won't be logged against self.assertRaises(AuthenticationError, @@ -282,7 +282,7 @@ def test_copy_to_system_source(self): "make sure we can 'convert' an LDAP user into a system one" with self.admin_access.repo_cnx() as cnx: - source = self.repo.sources_by_uri['ldap'] + source = self.repo.source_by_uri('ldap') eid = cnx.execute('CWUser X WHERE X login %(login)s', {'login': 'syt'})[0][0] cnx.execute('SET X cw_source S WHERE X eid %(x)s, S name "system"', {'x': eid}) cnx.commit() @@ -315,7 +315,7 @@ def setup_database(self): with self.admin_access.repo_cnx() as cnx: - lfsource = cnx.repo.sources_by_uri['ldap'] + lfsource = cnx.repo.source_by_uri('ldap') del lfsource.user_attrs['userPassword'] super(LDAPGeneratePwdTC, self).setup_database() @@ -342,7 +342,7 @@ cnx.commit() with self.repo.internal_cnx() as cnx: self.pull(cnx) - repo_source = self.repo.sources_by_uri['ldap'] + repo_source = self.repo.source_by_uri('ldap') self.assertRaises(AuthenticationError, repo_source.authenticate, cnx, 'syt', 'syt') with self.admin_access.repo_cnx() as cnx: @@ -374,7 +374,7 @@ self.delete_ldap_entry('uid=syt,ou=People,dc=cubicweb,dc=test') with self.repo.internal_cnx() as cnx: self.pull(cnx) - source = self.repo.sources_by_uri['ldap'] + source = self.repo.source_by_uri('ldap') self.assertRaises(AuthenticationError, source.authenticate, cnx, 'syt', 'syt') with self.admin_access.repo_cnx() as cnx: @@ -413,7 +413,7 @@ # test reactivating BY HAND the user isn't enough to # authenticate, as the native source refuse to authenticate # user from other sources - repo_source = self.repo.sources_by_uri['ldap'] + repo_source = self.repo.source_by_uri('ldap') self.delete_ldap_entry('uid=syt,ou=People,dc=cubicweb,dc=test') with self.repo.internal_cnx() as cnx: self.pull(cnx) diff -r 752b94ed9748 -r d540defa0591 cubicweb/server/test/unittest_postgres.py --- a/cubicweb/server/test/unittest_postgres.py Tue Apr 04 17:43:56 2017 +0200 +++ b/cubicweb/server/test/unittest_postgres.py Wed Apr 05 14:59:09 2017 +0200 @@ -52,7 +52,7 @@ def test_eid_range(self): # concurrent allocation of eid ranges - source = self.repo.sources_by_uri['system'] + source = self.repo.system_source range1 = [] range2 = [] def allocate_eid_ranges(session, target): diff -r 752b94ed9748 -r d540defa0591 cubicweb/server/test/unittest_undo.py --- a/cubicweb/server/test/unittest_undo.py Tue Apr 04 17:43:56 2017 +0200 +++ b/cubicweb/server/test/unittest_undo.py Wed Apr 05 14:59:09 2017 +0200 @@ -378,7 +378,7 @@ txuuid = cnx.commit() p = cnx.create_entity('Personne', nom=u'louis', fiche=c) cnx.commit() - integrityerror = self.repo.sources_by_uri['system'].dbhelper.dbapi_module.IntegrityError + integrityerror = self.repo.system_source.dbhelper.dbapi_module.IntegrityError with self.assertRaises(integrityerror): cnx.undo_transaction(txuuid) diff -r 752b94ed9748 -r d540defa0591 cubicweb/sobjects/services.py --- a/cubicweb/sobjects/services.py Tue Apr 04 17:43:56 2017 +0200 +++ b/cubicweb/sobjects/services.py Wed Apr 05 14:59:09 2017 +0200 @@ -142,6 +142,6 @@ __select__ = Service.__select__ & match_user_groups('managers') def call(self, source_eid): - source = self._cw.repo.sources_by_eid[source_eid] + source = self._cw.repo.source_by_eid(source_eid) result = source.pull_data(self._cw, force=True, async=True) return result['import_log_eid']