311 for mappedurl in URL_MAPPING: |
311 for mappedurl in URL_MAPPING: |
312 if url.startswith(mappedurl): |
312 if url.startswith(mappedurl): |
313 return url.replace(mappedurl, URL_MAPPING[mappedurl], 1) |
313 return url.replace(mappedurl, URL_MAPPING[mappedurl], 1) |
314 return url |
314 return url |
315 |
315 |
316 def retrieve_url(self, url): |
316 def retrieve_url(self, url, data=None, headers=None): |
317 """Return stream linked by the given url: |
317 """Return stream linked by the given url: |
318 * HTTP urls will be normalized (see :meth:`normalize_url`) |
318 * HTTP urls will be normalized (see :meth:`normalize_url`) |
319 * handle file:// URL |
319 * handle file:// URL |
320 * other will be considered as plain content, useful for testing purpose |
320 * other will be considered as plain content, useful for testing purpose |
321 """ |
321 """ |
322 if url.startswith('http'): |
322 if url.startswith('http'): |
323 url = self.normalize_url(url) |
323 url = self.normalize_url(url) |
324 self.source.info('GET %s', url) |
324 if data: |
325 return _OPENER.open(url, timeout=self.source.http_timeout) |
325 self.source.info('POST %s %s', url, data) |
|
326 else: |
|
327 self.source.info('GET %s', url) |
|
328 req = urllib2.Request(url, data, headers) |
|
329 return _OPENER.open(req, timeout=self.source.http_timeout) |
326 if url.startswith('file://'): |
330 if url.startswith('file://'): |
327 return URLLibResponseAdapter(open(url[7:]), url) |
331 return URLLibResponseAdapter(open(url[7:]), url) |
328 return URLLibResponseAdapter(StringIO.StringIO(url), url) |
332 return URLLibResponseAdapter(StringIO.StringIO(url), url) |
329 |
333 |
330 def add_schema_config(self, schemacfg, checkonly=False): |
334 def add_schema_config(self, schemacfg, checkonly=False): |