equal
deleted
inserted
replaced
22 import urllib2 |
22 import urllib2 |
23 import StringIO |
23 import StringIO |
24 from os.path import exists |
24 from os.path import exists |
25 from datetime import datetime, timedelta |
25 from datetime import datetime, timedelta |
26 from cookielib import CookieJar |
26 from cookielib import CookieJar |
27 import urlparse |
27 |
|
28 from six.moves.urllib.parse import urlparse |
|
29 |
28 from lxml import etree |
30 from lxml import etree |
29 |
31 |
30 from cubicweb import RegistryNotFound, ObjectNotFound, ValidationError, UnknownEid |
32 from cubicweb import RegistryNotFound, ObjectNotFound, ValidationError, UnknownEid |
31 from cubicweb.server.repository import preprocess_inlined_relations |
33 from cubicweb.server.repository import preprocess_inlined_relations |
32 from cubicweb.server.sources import AbstractSource |
34 from cubicweb.server.sources import AbstractSource |
325 * other will be considered as plain content, useful for testing purpose |
327 * other will be considered as plain content, useful for testing purpose |
326 |
328 |
327 For http URLs, it will try to find a cwclientlib config entry |
329 For http URLs, it will try to find a cwclientlib config entry |
328 (if available) and use it as requester. |
330 (if available) and use it as requester. |
329 """ |
331 """ |
330 purl = urlparse.urlparse(url) |
332 purl = urlparse(url) |
331 if purl.scheme == 'file': |
333 if purl.scheme == 'file': |
332 return URLLibResponseAdapter(open(url[7:]), url) |
334 return URLLibResponseAdapter(open(url[7:]), url) |
333 |
335 |
334 url = self.normalize_url(url) |
336 url = self.normalize_url(url) |
335 |
337 |
527 # ValueError: no config entry found |
529 # ValueError: no config entry found |
528 # EnvironmentError: no cwclientlib config file found |
530 # EnvironmentError: no cwclientlib config file found |
529 self.source.debug(str(exc)) |
531 self.source.debug(str(exc)) |
530 |
532 |
531 # no chance with cwclientlib, fall back to former implementation |
533 # no chance with cwclientlib, fall back to former implementation |
532 if urlparse.urlparse(url).scheme in ('http', 'https'): |
534 if urlparse(url).scheme in ('http', 'https'): |
533 try: |
535 try: |
534 _OPENER.open(url, timeout=self.source.http_timeout) |
536 _OPENER.open(url, timeout=self.source.http_timeout) |
535 except urllib2.HTTPError as ex: |
537 except urllib2.HTTPError as ex: |
536 if ex.code == 404: |
538 if ex.code == 404: |
537 return True |
539 return True |