# HG changeset patch # User sylvain.thenault@logilab.fr # Date 1233242793 -3600 # Node ID 06390418cd9a9057f4639b0eb3c5e3d4c06bf740 # Parent b6003d74a600303488cfff28253a2bbd182091bd pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri) diff -r b6003d74a600 -r 06390418cd9a server/sources/pyrorql.py --- a/server/sources/pyrorql.py Thu Jan 29 15:26:09 2009 +0100 +++ b/server/sources/pyrorql.py Thu Jan 29 16:26:33 2009 +0100 @@ -159,7 +159,8 @@ external repository """ self.info('synchronizing pyro source %s', self.uri) - extrepo = self.get_connection()._repo + cnx = self.get_connection() + extrepo = cnx._repo etypes = self.support_entities.keys() if mtime is None: mtime = self.last_update_time() @@ -170,11 +171,13 @@ try: for etype, extid in modified: try: - eid = self.extid2eid(extid, etype, session) - rset = session.eid_rset(eid, etype) - entity = rset.get_entity(0, 0) - entity.complete(entity.e_schema.indexable_attributes()) - repo.index_entity(session, entity) + exturi = cnx.describe(extid)[1] + if exturi == 'system' or not exturi in repo.sources_by_uri: + eid = self.extid2eid(extid, etype, session) + rset = session.eid_rset(eid, etype) + entity = rset.get_entity(0, 0) + entity.complete(entity.e_schema.indexable_attributes()) + repo.index_entity(session, entity) except: self.exception('while updating %s with external id %s of source %s', etype, extid, self.uri) @@ -277,18 +280,29 @@ descr = rset.description if rset: needtranslation = [] + rows = rset.rows for i, etype in enumerate(descr[0]): if (etype is None or not self.schema.eschema(etype).is_final() or getattr(union.locate_subquery(i, etype, args).selection[i], 'uidtype', None)): needtranslation.append(i) if needtranslation: - for rowindex, row in enumerate(rset): + cnx = session.pool.connection(self.uri) + for rowindex in xrange(rset.rowcount - 1, -1, -1): + row = rows[rowindex] for colindex in needtranslation: if row[colindex] is not None: # optional variable etype = descr[rowindex][colindex] - eid = self.extid2eid(row[colindex], etype, session) - row[colindex] = eid - results = rset.rows + exttype, exturi, extid = cnx.describe(row[colindex]) + if exturi == 'system' or not exturi in self.repo.sources_by_uri: + eid = self.extid2eid(row[colindex], etype, session) + row[colindex] = eid + else: + # skip this row + print 'skip external', etype, exturi, extid, self.uri + del rows[rowindex] + del descr[rowindex] + break + results = rows else: results = [] if server.DEBUG: diff -r b6003d74a600 -r 06390418cd9a server/test/data/sources_multi --- a/server/test/data/sources_multi Thu Jan 29 15:26:09 2009 +0100 +++ b/server/test/data/sources_multi Thu Jan 29 16:26:33 2009 +0100 @@ -15,6 +15,13 @@ cubicweb-password = gingkow mapping-file = extern_mapping.py +[extern-multi] +adapter = pyrorql +pyro-ns-id = extern-multi +cubicweb-user = admin +cubicweb-password = gingkow +mapping-file = extern_mapping.py + [admin] login = admin password = gingkow diff -r b6003d74a600 -r 06390418cd9a server/test/unittest_multisources.py --- a/server/test/unittest_multisources.py Thu Jan 29 15:26:09 2009 +0100 +++ b/server/test/unittest_multisources.py Thu Jan 29 16:26:33 2009 +0100 @@ -10,10 +10,13 @@ sourcefile = 'sources_multi' -class ExternalSourceConfiguration(TestServerConfiguration): +class ExternalSource1Configuration(TestServerConfiguration): sourcefile = 'sources_extern' + +class ExternalSource2Configuration(TestServerConfiguration): + sourcefile = 'sources_multi2' -repo2, cnx2 = init_test_database('sqlite', config=ExternalSourceConfiguration('data')) +repo2, cnx2 = init_test_database('sqlite', config=ExternalSource1Configuration('data')) cu = cnx2.cursor() ec1 = cu.execute('INSERT Card X: X title "C3: An external card", X wikiid "aaa"')[0][0] cu.execute('INSERT Card X: X title "C4: Ze external card", X wikiid "zzz"') @@ -22,9 +25,11 @@ MTIME = now() - 0.1 +repo3, cnx3 = init_test_database('sqlite', config=ExternalSource2Configuration('data')) + # XXX, access existing connection, no pyro connection from cubicweb.server.sources.pyrorql import PyroRQLSource -PyroRQLSource.get_connection = lambda x: cnx2 +PyroRQLSource.get_connection = lambda x: x.uri == 'extern-multi' and cnx3 or cnx2 # necessary since the repository is closing its initial connections pool though # we want to keep cnx2 valid from cubicweb.dbapi import Connection @@ -47,6 +52,11 @@ self.ic2 = self.execute('INSERT Card X: X title "C2: Ze internal card", X wikiid "zzzi"')[0][0] self.commit() do_monkey_patch() + print 'main repo', self.repo + print 'external simple repo', repo2 + print 'external multi repo', repo3 + print 'ic1', self.ic1 + print 'ic2', self.ic2 def tearDown(self): RepositoryBasedTC.tearDown(self) @@ -84,12 +94,12 @@ self.assertEquals(rset.rows, rsetbase.rows[2:4]) def test_has_text(self): - self.repo.sources[-1].synchronize(MTIME) # in case fti_update has been run before + self.repo.sources_by_uri['extern'].synchronize(MTIME) # in case fti_update has been run before self.failUnless(self.execute('Any X WHERE X has_text "affref"')) self.failUnless(self.execute('Affaire X WHERE X has_text "affref"')) def test_anon_has_text(self): - self.repo.sources[-1].synchronize(MTIME) # in case fti_update has been run before + self.repo.sources_by_uri['extern'].synchronize(MTIME) # in case fti_update has been run before self.execute('INSERT Affaire X: X ref "no readable card"')[0][0] aff1 = self.execute('INSERT Affaire X: X ref "card"')[0][0] # grant read access @@ -97,8 +107,9 @@ self.commit() cnx = self.login('anon') cu = cnx.cursor() + print '*'*80 rset = cu.execute('Any X WHERE X has_text "card"') - self.assertEquals(len(rset), 5) + self.assertEquals(len(rset), 5, zip(rset.rows, rset.description)) def test_synchronization(self): cu = cnx2.cursor() @@ -107,13 +118,18 @@ cnx2.commit() try: # force sync - self.repo.sources[-1].synchronize(MTIME) + self.repo.sources_by_uri['extern'].synchronize(MTIME) self.failUnless(self.execute('Any X WHERE X has_text "blah"')) self.failUnless(self.execute('Any X WHERE X has_text "affreux"')) + print 'delete', aff2 cu.execute('DELETE Affaire X WHERE X eid %(x)s', {'x': aff2}) cnx2.commit() - self.repo.sources[-1].synchronize(MTIME) - self.failIf(self.execute('Any X WHERE X has_text "affreux"')) + print 'sync' + self.repo.sources_by_uri['extern'].synchronize(MTIME) + print 'query' + rset = self.execute('Any X WHERE X has_text "affreux"') + print zip(rset.rows, rset.description) + self.failIf(rset) finally: # restore state cu.execute('SET X ref "AFFREF" WHERE X eid %(x)s', {'x': aff1}, 'x') @@ -139,8 +155,10 @@ self.execute('Any X ORDERBY DUMB_SORT(RF) WHERE X title RF') def test_in_eid(self): - iec1 = self.repo.extid2eid(self.repo.sources[-1], ec1, 'Card', self.session) + iec1 = self.repo.extid2eid(self.repo.sources_by_uri['extern'], ec1, + 'Card', self.session) rset = self.execute('Any X WHERE X eid IN (%s, %s)' % (iec1, self.ic1)) + print 'hop', iec1, self.ic1 self.assertEquals(sorted(r[0] for r in rset.rows), sorted([iec1, self.ic1])) def test_greater_eid(self):