322 if url.startswith('http'): |
322 if url.startswith('http'): |
323 url = self.normalize_url(url) |
323 url = self.normalize_url(url) |
324 self.source.info('GET %s', url) |
324 self.source.info('GET %s', url) |
325 return _OPENER.open(url, timeout=self.source.http_timeout) |
325 return _OPENER.open(url, timeout=self.source.http_timeout) |
326 if url.startswith('file://'): |
326 if url.startswith('file://'): |
327 return open(url[7:]) |
327 return URLLibResponseAdapter(open(url[7:]), url) |
328 return StringIO.StringIO(url) |
328 return URLLibResponseAdapter(StringIO.StringIO(url), url) |
329 |
329 |
330 def add_schema_config(self, schemacfg, checkonly=False): |
330 def add_schema_config(self, schemacfg, checkonly=False): |
331 """added CWSourceSchemaConfig, modify mapping accordingly""" |
331 """added CWSourceSchemaConfig, modify mapping accordingly""" |
332 msg = schemacfg._cw._("this parser doesn't use a mapping") |
332 msg = schemacfg._cw._("this parser doesn't use a mapping") |
333 raise ValidationError(schemacfg.eid, {None: msg}) |
333 raise ValidationError(schemacfg.eid, {None: msg}) |
488 return True |
488 return True |
489 elif extid.startswith('file://'): |
489 elif extid.startswith('file://'): |
490 return exists(extid[7:]) |
490 return exists(extid[7:]) |
491 return False |
491 return False |
492 |
492 |
|
493 |
|
494 class URLLibResponseAdapter(object): |
|
495 """Thin wrapper to be used to fake a value returned by urllib2.urlopen""" |
|
496 def __init__(self, stream, url, code=200): |
|
497 self._stream = stream |
|
498 self._url = url |
|
499 self.code = code |
|
500 |
|
501 def read(self, *args): |
|
502 return self._stream.read(*args) |
|
503 |
|
504 def geturl(self): |
|
505 return self._url |
|
506 |
|
507 def getcode(self): |
|
508 return self.code |
|
509 |
|
510 def info(self): |
|
511 from mimetools import Message |
|
512 return Message(StringIO.StringIO()) |
|
513 |
493 # use a cookie enabled opener to use session cookie if any |
514 # use a cookie enabled opener to use session cookie if any |
494 _OPENER = urllib2.build_opener() |
515 _OPENER = urllib2.build_opener() |
495 try: |
516 try: |
496 from logilab.common import urllib2ext |
517 from logilab.common import urllib2ext |
497 _OPENER.add_handler(urllib2ext.HTTPGssapiAuthHandler()) |
518 _OPENER.add_handler(urllib2ext.HTTPGssapiAuthHandler()) |