[datafeed parser] enhance retrieve_url to support POSTing data and custom HTTP headers
authorSylvain Thénault <sylvain.thenault@logilab.fr>
Fri, 21 Mar 2014 08:28:32 +0100
changeset 9825 946b483bc8a1
parent 9824 30183ecf5c61
child 9826 7c17659c9eae
[datafeed parser] enhance retrieve_url to support POSTing data and custom HTTP headers Closes #3682069
server/sources/datafeed.py
--- a/server/sources/datafeed.py	Tue Mar 25 09:20:37 2014 +0100
+++ b/server/sources/datafeed.py	Fri Mar 21 08:28:32 2014 +0100
@@ -313,7 +313,7 @@
                 return url.replace(mappedurl, URL_MAPPING[mappedurl], 1)
         return url
 
-    def retrieve_url(self, url):
+    def retrieve_url(self, url, data=None, headers=None):
         """Return stream linked by the given url:
         * HTTP urls will be normalized (see :meth:`normalize_url`)
         * handle file:// URL
@@ -321,8 +321,12 @@
         """
         if url.startswith('http'):
             url = self.normalize_url(url)
-            self.source.info('GET %s', url)
-            return _OPENER.open(url, timeout=self.source.http_timeout)
+            if data:
+                self.source.info('POST %s %s', url, data)
+            else:
+                self.source.info('GET %s', url)
+            req = urllib2.Request(url, data, headers)
+            return _OPENER.open(req, timeout=self.source.http_timeout)
         if url.startswith('file://'):
             return URLLibResponseAdapter(open(url[7:]), url)
         return URLLibResponseAdapter(StringIO.StringIO(url), url)