equal
deleted
inserted
replaced
136 parser = self._get_parser(session, sourceuris=myuris) |
136 parser = self._get_parser(session, sourceuris=myuris) |
137 error = False |
137 error = False |
138 self.info('pulling data for source %s', self.uri) |
138 self.info('pulling data for source %s', self.uri) |
139 for url in self.urls: |
139 for url in self.urls: |
140 try: |
140 try: |
141 if parser.process(url): |
141 if parser.process(url, raise_on_error): |
142 error = True |
142 error = True |
143 except IOError, exc: |
143 except IOError, exc: |
144 if raise_on_error: |
144 if raise_on_error: |
145 raise |
145 raise |
146 self.error('could not pull data while processing %s: %s', |
146 self.error('could not pull data while processing %s: %s', |
245 return self.stats['updated'].add(entity.eid) |
245 return self.stats['updated'].add(entity.eid) |
246 |
246 |
247 |
247 |
248 class DataFeedXMLParser(DataFeedParser): |
248 class DataFeedXMLParser(DataFeedParser): |
249 |
249 |
250 def process(self, url, partialcommit=True): |
250 def process(self, url, raise_on_error=False, partialcommit=True): |
251 """IDataFeedParser main entry point""" |
251 """IDataFeedParser main entry point""" |
252 error = False |
252 error = False |
253 for args in self.parse(url): |
253 for args in self.parse(url): |
254 try: |
254 try: |
255 self.process_item(*args) |
255 self.process_item(*args) |
257 # commit+set_pool instead of commit(reset_pool=False) to let |
257 # commit+set_pool instead of commit(reset_pool=False) to let |
258 # other a chance to get our pool |
258 # other a chance to get our pool |
259 self._cw.commit() |
259 self._cw.commit() |
260 self._cw.set_pool() |
260 self._cw.set_pool() |
261 except ValidationError, exc: |
261 except ValidationError, exc: |
|
262 if raise_on_error: |
|
263 raise |
262 if partialcommit: |
264 if partialcommit: |
263 self.source.error('Skipping %s because of validation error %s' % (args, exc)) |
265 self.source.error('Skipping %s because of validation error %s' % (args, exc)) |
264 self._cw.rollback() |
266 self._cw.rollback() |
265 self._cw.set_pool() |
267 self._cw.set_pool() |
266 error = True |
268 error = True |