[etwist] fix handling of multiple files per field
authorAurelien Campeas <aurelien.campeas@logilab.fr>
Mon, 24 Jun 2013 19:00:40 +0200
changeset 9147 01124cfd4b1f
parent 9146 9b58a6406a64
child 9148 1b549c1acd4f
[etwist] fix handling of multiple files per field html5 permits multiple files uploads, which can be expressed as:: <input type='file' multiple='multiple' /> This changeset avoids previous crash. Nothing is changed when a single file is uploaded (backward compat is thus preserved). When multiple files are uploaded for a single html input tag, the corresponding web request form key receives a list of tuples like [('filename-1', IStream1), ('filename-2', IStream2), ...]. closes #2847207.
etwist/request.py
etwist/server.py
web/formfields.py
web/test/data/views.py
web/test/unittest_web.py
--- a/etwist/request.py	Tue Jul 09 15:58:26 2013 +0200
+++ b/etwist/request.py	Mon Jun 24 19:00:40 2013 +0200
@@ -24,15 +24,21 @@
 
 
 class CubicWebTwistedRequestAdapter(CubicWebRequestBase):
+    """ from twisted .req to cubicweb .form
+    req.files are put into .form[<filefield>]
+    """
     def __init__(self, req, vreg, https):
         self._twreq = req
         super(CubicWebTwistedRequestAdapter, self).__init__(
             vreg, https, req.args, headers=req.received_headers)
-        for key, (name, stream) in req.files.iteritems():
-            if name is None:
-                self.form[key] = (name, stream)
-            else:
-                self.form[key] = (unicode(name, self.encoding), stream)
+        for key, name_stream_list in req.files.iteritems():
+            for name, stream in name_stream_list:
+                if name is not None:
+                    name = unicode(name, self.encoding)
+                self.form.setdefault(key, []).append((name, stream))
+            # 3.16.4 backward compat
+            if len(self.form[key]) == 1:
+                self.form[key] = self.form[key][0]
         self.content = self._twreq.content # stream
 
     def http_method(self):
--- a/etwist/server.py	Tue Jul 09 15:58:26 2013 +0200
+++ b/etwist/server.py	Mon Jun 24 19:00:40 2013 +0200
@@ -244,7 +244,6 @@
             self._do_process_multipart = True
     self.process()
 
-
 @monkeypatch(http.Request)
 def process_multipart(self):
     if not self._do_process_multipart:
@@ -254,16 +253,17 @@
                         keep_blank_values=1,
                         strict_parsing=1)
     for key in form:
-        value = form[key]
-        if isinstance(value, list):
-            self.args[key] = [v.value for v in value]
-        elif value.filename:
-            if value.done != -1: # -1 is transfer has been interrupted
-                self.files[key] = (value.filename, value.file)
+        values = form[key]
+        if not isinstance(values, list):
+            values = [values]
+        for value in values:
+            if value.filename:
+                if value.done != -1: # -1 is transfer has been interrupted
+                    self.files.setdefault(key, []).append((value.filename, value.file))
+                else:
+                    self.files.setdefault(key, []).append((None, None))
             else:
-                self.files[key] = (None, None)
-        else:
-            self.args[key] = value.value
+                self.args.setdefault(key, []).append(value.value)
 
 from logging import getLogger
 from cubicweb import set_log_methods
--- a/web/formfields.py	Tue Jul 09 15:58:26 2013 +0200
+++ b/web/formfields.py	Mon Jun 24 19:00:40 2013 +0200
@@ -755,8 +755,13 @@
             # raise UnmodifiedField instead of returning None, since the later
             # will try to remove already attached file if any
             raise UnmodifiedField()
-        # value is a 2-uple (filename, stream)
+        # value is a 2-uple (filename, stream) or a list of such
+        # tuples (multiple files)
         try:
+            if isinstance(value, list):
+                value = value[0]
+                form.warning('mutiple files provided, however '
+                             'only the first will be picked')
             filename, stream = value
         except ValueError:
             raise UnmodifiedField()
--- a/web/test/data/views.py	Tue Jul 09 15:58:26 2013 +0200
+++ b/web/test/data/views.py	Mon Jun 24 19:00:40 2013 +0200
@@ -18,6 +18,7 @@
 
 from cubicweb.web import Redirect
 from cubicweb.web.application import CubicWebPublisher
+from cubicweb.web.views.ajaxcontroller import ajaxfunc
 
 # proof of concept : monkey patch handle method so that if we are in an
 # anonymous session and __fblogin is found is req.form, the user with the
@@ -40,5 +41,35 @@
         assert req.user.login == login
     return orig_handle(self, req, path)
 
+
+def _recursive_replace_stream_by_content(tree):
+    """ Search for streams (i.e. object that have a 'read' method) in a tree
+    (which branches are lists or tuples), and substitute them by their content,
+    leaving other leafs identical. A copy of the tree with only lists as
+    branches is returned.
+    """
+    if not isinstance(tree, (list, tuple)):
+        if hasattr(tree, 'read'):
+            return tree.read()
+        return tree
+    else:
+        return [_recursive_replace_stream_by_content(value)
+                for value in tree]            
+
+
+@ajaxfunc(output_type='json')
+def fileupload(self):
+    """ Return a json copy of the web request formin which uploaded files
+    are read and their content substitute the received streams.
+    """
+    try:
+        result_dict = {}
+        for key, value in self._cw.form.iteritems():
+            result_dict[key] = _recursive_replace_stream_by_content(value)
+        return result_dict
+    except Exception, ex:
+        import traceback as tb
+        tb.print_exc(ex)
+
 orig_handle = CubicWebPublisher.main_handle_request
 CubicWebPublisher.main_handle_request = auto_login_handle_request
--- a/web/test/unittest_web.py	Tue Jul 09 15:58:26 2013 +0200
+++ b/web/test/unittest_web.py	Mon Jun 24 19:00:40 2013 +0200
@@ -16,7 +16,17 @@
 # You should have received a copy of the GNU Lesser General Public License along
 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
 
+from json import loads
+from os.path import join
+
+try:
+    import requests
+    assert [int(n) for n in requests.__version__.split('.', 2)][:2] >= [1, 2]
+except (ImportError, AssertionError):
+    requests = None
+
 from logilab.common.testlib import TestCase, unittest_main
+from cubicweb.devtools.httptest import CubicWebServerTC
 from cubicweb.devtools.fake import FakeRequest
 
 class AjaxReplaceUrlTC(TestCase):
@@ -43,5 +53,45 @@
             (cbname, qs, req.pageid),
             req.html_headers.post_inlined_scripts[0])
 
+
+class FileUploadTC(CubicWebServerTC):
+
+    def setUp(self):
+        "Skip whole test class if a suitable requests module is not available"
+        if requests is None:
+            self.skipTest('Python ``requests`` module is not available')
+        super(FileUploadTC, self).setUp()
+
+    @property
+    def _post_url(self):
+        return self.request().build_url('ajax', fname='fileupload')
+
+    def _fobject(self, fname):
+        return open(join(self.datadir, fname), 'rb')
+
+    def _fcontent(self, fname):
+        return self._fobject(fname).read()
+
+    def test_single_file_upload(self):
+        files = {'file': ('schema.py', self._fobject('schema.py'))}
+        webreq = requests.post(self._post_url, files=files)
+        # check backward compat : a single uploaded file leads to a single
+        # 2-uple in the request form
+        expect = {'fname': u'fileupload',
+                  'file': ['schema.py', self._fcontent('schema.py')]}
+        self.assertEqual(webreq.status_code, 200)
+        self.assertDictEqual(expect, loads(webreq.content))
+
+    def test_multiple_file_upload(self):
+        files = [('files', ('schema.py', self._fobject('schema.py'))),
+                 ('files', ('views.py',  self._fobject('views.py')))]
+        webreq = requests.post(self._post_url, files=files,)
+        expect = {'fname': u'fileupload',
+                  'files': [['schema.py', self._fcontent('schema.py')],
+                            ['views.py', self._fcontent('views.py')]],}
+        self.assertEqual(webreq.status_code, 200)
+        self.assertDictEqual(expect, loads(webreq.content))
+
+
 if __name__ == '__main__':
     unittest_main()