closes #1793991: [datafeed] error handling: log unexpected exceptions / don't stop on first validation error
authorSylvain Thénault <sylvain.thenault@logilab.fr>
Thu, 30 Jun 2011 16:26:37 +0200
changeset 7590 a9aad6c25836
parent 7588 7588240bcb36
child 7591 cb6ece9cdb78
closes #1793991: [datafeed] error handling: log unexpected exceptions / don't stop on first validation error
server/sources/datafeed.py
--- a/server/sources/datafeed.py	Thu Jun 30 13:38:30 2011 +0200
+++ b/server/sources/datafeed.py	Thu Jun 30 16:26:37 2011 +0200
@@ -192,12 +192,18 @@
             try:
                 if parser.process(url, raise_on_error):
                     error = True
-            except Exception, exc:
+            except IOError, exc:
                 if raise_on_error:
                     raise
                 self.error('could not pull data while processing %s: %s',
                            url, exc)
                 error = True
+            except Exception, exc:
+                if raise_on_error:
+                    raise
+                self.exception('error while processing %s: %s',
+                               url, exc)
+                error = True
         return error
 
     def before_entity_insertion(self, session, lid, etype, eid, sourceparams):
@@ -266,8 +272,12 @@
         source = self._cw.repo.sources_by_uri.get(
             sourceparams.pop('cwsource', None), self.source)
         sourceparams['parser'] = self
-        eid = source.extid2eid(str(uri), etype, self._cw,
-                               sourceparams=sourceparams)
+        try:
+            eid = source.extid2eid(str(uri), etype, self._cw,
+                                   sourceparams=sourceparams)
+        except ValidationError, ex:
+            self.source.error('error while creating %s: %s', etype, ex)
+            return None
         if eid < 0:
             # entity has been moved away from its original source
             #