equal
deleted
inserted
replaced
27 from base64 import b64decode |
27 from base64 import b64decode |
28 |
28 |
29 from Pyro.errors import PyroError, ConnectionClosedError |
29 from Pyro.errors import PyroError, ConnectionClosedError |
30 |
30 |
31 from logilab.common.configuration import REQUIRED |
31 from logilab.common.configuration import REQUIRED |
|
32 from logilab.common.optik_ext import check_yn |
32 |
33 |
33 from rql.nodes import Constant |
34 from rql.nodes import Constant |
34 from rql.utils import rqlvar_maker |
35 from rql.utils import rqlvar_maker |
35 |
36 |
36 from cubicweb import dbapi, server |
37 from cubicweb import dbapi, server |
117 {'type' : 'string', |
118 {'type' : 'string', |
118 'default': '', |
119 'default': '', |
119 'help': 'url of the web site for the distant repository, if you want ' |
120 'help': 'url of the web site for the distant repository, if you want ' |
120 'to generate external link to entities from this repository', |
121 'to generate external link to entities from this repository', |
121 'group': 'pyro-source', 'level': 1, |
122 'group': 'pyro-source', 'level': 1, |
|
123 }), |
|
124 ('skip-external-entities', |
|
125 {'type' : 'yn', |
|
126 'default': False, |
|
127 'help': 'should entities not local to the source be considered or not', |
|
128 'group': 'pyro-source', 'level': 0, |
122 }), |
129 }), |
123 ('pyro-ns-host', |
130 ('pyro-ns-host', |
124 {'type' : 'string', |
131 {'type' : 'string', |
125 'default': None, |
132 'default': None, |
126 'help': 'Pyro name server\'s host. If not set, default to the value \ |
133 'help': 'Pyro name server\'s host. If not set, default to the value \ |
177 'help': _('timestamp of the latest source synchronization.'), |
184 'help': _('timestamp of the latest source synchronization.'), |
178 'group': 'sources', |
185 'group': 'sources', |
179 }),) |
186 }),) |
180 register_persistent_options(myoptions) |
187 register_persistent_options(myoptions) |
181 self._query_cache = TimedCache(1800) |
188 self._query_cache = TimedCache(1800) |
|
189 self._skip_externals = check_yn(None, 'skip-external-entities', |
|
190 source_config.get('skip-external-entities', False)) |
182 |
191 |
183 def reset_caches(self): |
192 def reset_caches(self): |
184 """method called during test to reset potential source caches""" |
193 """method called during test to reset potential source caches""" |
185 self._query_cache = TimedCache(1800) |
194 self._query_cache = TimedCache(1800) |
186 |
195 |
207 """method called by the repository once ready to handle request""" |
216 """method called by the repository once ready to handle request""" |
208 interval = int(self.config.get('synchronization-interval', 5*60)) |
217 interval = int(self.config.get('synchronization-interval', 5*60)) |
209 self.repo.looping_task(interval, self.synchronize) |
218 self.repo.looping_task(interval, self.synchronize) |
210 self.repo.looping_task(self._query_cache.ttl.seconds/10, |
219 self.repo.looping_task(self._query_cache.ttl.seconds/10, |
211 self._query_cache.clear_expired) |
220 self._query_cache.clear_expired) |
|
221 |
|
222 def map_entity_source(self, exturi): |
|
223 return (exturi == 'system' or |
|
224 not (exturi in self.repo.sources_by_uri or self._skip_externals)) |
212 |
225 |
213 def synchronize(self, mtime=None): |
226 def synchronize(self, mtime=None): |
214 """synchronize content known by this repository with content in the |
227 """synchronize content known by this repository with content in the |
215 external repository |
228 external repository |
216 """ |
229 """ |
233 source = repo.system_source |
246 source = repo.system_source |
234 try: |
247 try: |
235 for etype, extid in modified: |
248 for etype, extid in modified: |
236 try: |
249 try: |
237 exturi = cnx.describe(extid)[1] |
250 exturi = cnx.describe(extid)[1] |
238 if exturi == 'system' or not exturi in repo.sources_by_uri: |
251 if self.map_entity_source(exturi): |
239 eid = self.extid2eid(str(extid), etype, session) |
252 eid = self.extid2eid(str(extid), etype, session) |
240 rset = session.eid_rset(eid, etype) |
253 rset = session.eid_rset(eid, etype) |
241 entity = rset.get_entity(0, 0) |
254 entity = rset.get_entity(0, 0) |
242 entity.complete(entity.e_schema.indexable_attributes()) |
255 entity.complete(entity.e_schema.indexable_attributes()) |
243 source.index_entity(session, entity) |
256 source.index_entity(session, entity) |
362 row = rows[rowindex] |
375 row = rows[rowindex] |
363 for colindex in needtranslation: |
376 for colindex in needtranslation: |
364 if row[colindex] is not None: # optional variable |
377 if row[colindex] is not None: # optional variable |
365 etype = descr[rowindex][colindex] |
378 etype = descr[rowindex][colindex] |
366 exttype, exturi, extid = cnx.describe(row[colindex]) |
379 exttype, exturi, extid = cnx.describe(row[colindex]) |
367 if exturi == 'system' or not exturi in self.repo.sources_by_uri: |
380 if self.map_entity_source(exturi): |
368 eid = self.extid2eid(str(row[colindex]), etype, |
381 eid = self.extid2eid(str(row[colindex]), etype, |
369 session) |
382 session) |
370 row[colindex] = eid |
383 row[colindex] = eid |
371 else: |
384 else: |
372 # skip this row |
385 # skip this row |