equal
deleted
inserted
replaced
118 def fresh(self): |
118 def fresh(self): |
119 if self.latest_retrieval is None: |
119 if self.latest_retrieval is None: |
120 return False |
120 return False |
121 return datetime.now() < (self.latest_retrieval + self.synchro_interval) |
121 return datetime.now() < (self.latest_retrieval + self.synchro_interval) |
122 |
122 |
123 def pull_data(self, session, force=False): |
123 def pull_data(self, session, force=False, raise_on_error=False): |
124 if not force and self.fresh(): |
124 if not force and self.fresh(): |
125 return {} |
125 return {} |
126 if self.config['delete-entities']: |
126 if self.config['delete-entities']: |
127 myuris = self.source_cwuris(session) |
127 myuris = self.source_cwuris(session) |
128 else: |
128 else: |
133 for url in self.urls: |
133 for url in self.urls: |
134 try: |
134 try: |
135 if parser.process(url): |
135 if parser.process(url): |
136 error = True |
136 error = True |
137 except IOError, exc: |
137 except IOError, exc: |
|
138 if raise_on_error: |
|
139 raise |
138 self.error('could not pull data while processing %s: %s', |
140 self.error('could not pull data while processing %s: %s', |
139 url, exc) |
141 url, exc) |
140 error = True |
142 error = True |
141 if error: |
143 if error: |
142 self.warning("some error occured, don't attempt to delete entities") |
144 self.warning("some error occured, don't attempt to delete entities") |