[etwist] fix handling of multiple files per field
html5 permits multiple files uploads, which can be expressed as::
<input type='file' multiple='multiple' />
This changeset avoids previous crash. Nothing is changed when a single file is
uploaded (backward compat is thus preserved). When multiple files are uploaded
for a single html input tag, the corresponding web request form key receives a
list of tuples like [('filename-1', IStream1), ('filename-2', IStream2), ...].
closes #2847207.
--- a/etwist/request.py Tue Jul 09 15:58:26 2013 +0200
+++ b/etwist/request.py Mon Jun 24 19:00:40 2013 +0200
@@ -24,15 +24,21 @@
class CubicWebTwistedRequestAdapter(CubicWebRequestBase):
+ """ from twisted .req to cubicweb .form
+ req.files are put into .form[<filefield>]
+ """
def __init__(self, req, vreg, https):
self._twreq = req
super(CubicWebTwistedRequestAdapter, self).__init__(
vreg, https, req.args, headers=req.received_headers)
- for key, (name, stream) in req.files.iteritems():
- if name is None:
- self.form[key] = (name, stream)
- else:
- self.form[key] = (unicode(name, self.encoding), stream)
+ for key, name_stream_list in req.files.iteritems():
+ for name, stream in name_stream_list:
+ if name is not None:
+ name = unicode(name, self.encoding)
+ self.form.setdefault(key, []).append((name, stream))
+ # 3.16.4 backward compat
+ if len(self.form[key]) == 1:
+ self.form[key] = self.form[key][0]
self.content = self._twreq.content # stream
def http_method(self):
--- a/etwist/server.py Tue Jul 09 15:58:26 2013 +0200
+++ b/etwist/server.py Mon Jun 24 19:00:40 2013 +0200
@@ -244,7 +244,6 @@
self._do_process_multipart = True
self.process()
-
@monkeypatch(http.Request)
def process_multipart(self):
if not self._do_process_multipart:
@@ -254,16 +253,17 @@
keep_blank_values=1,
strict_parsing=1)
for key in form:
- value = form[key]
- if isinstance(value, list):
- self.args[key] = [v.value for v in value]
- elif value.filename:
- if value.done != -1: # -1 is transfer has been interrupted
- self.files[key] = (value.filename, value.file)
+ values = form[key]
+ if not isinstance(values, list):
+ values = [values]
+ for value in values:
+ if value.filename:
+ if value.done != -1: # -1 is transfer has been interrupted
+ self.files.setdefault(key, []).append((value.filename, value.file))
+ else:
+ self.files.setdefault(key, []).append((None, None))
else:
- self.files[key] = (None, None)
- else:
- self.args[key] = value.value
+ self.args.setdefault(key, []).append(value.value)
from logging import getLogger
from cubicweb import set_log_methods
--- a/web/formfields.py Tue Jul 09 15:58:26 2013 +0200
+++ b/web/formfields.py Mon Jun 24 19:00:40 2013 +0200
@@ -755,8 +755,13 @@
# raise UnmodifiedField instead of returning None, since the later
# will try to remove already attached file if any
raise UnmodifiedField()
- # value is a 2-uple (filename, stream)
+ # value is a 2-uple (filename, stream) or a list of such
+ # tuples (multiple files)
try:
+ if isinstance(value, list):
+ value = value[0]
+ form.warning('mutiple files provided, however '
+ 'only the first will be picked')
filename, stream = value
except ValueError:
raise UnmodifiedField()
--- a/web/test/data/views.py Tue Jul 09 15:58:26 2013 +0200
+++ b/web/test/data/views.py Mon Jun 24 19:00:40 2013 +0200
@@ -18,6 +18,7 @@
from cubicweb.web import Redirect
from cubicweb.web.application import CubicWebPublisher
+from cubicweb.web.views.ajaxcontroller import ajaxfunc
# proof of concept : monkey patch handle method so that if we are in an
# anonymous session and __fblogin is found is req.form, the user with the
@@ -40,5 +41,35 @@
assert req.user.login == login
return orig_handle(self, req, path)
+
+def _recursive_replace_stream_by_content(tree):
+ """ Search for streams (i.e. object that have a 'read' method) in a tree
+ (which branches are lists or tuples), and substitute them by their content,
+ leaving other leafs identical. A copy of the tree with only lists as
+ branches is returned.
+ """
+ if not isinstance(tree, (list, tuple)):
+ if hasattr(tree, 'read'):
+ return tree.read()
+ return tree
+ else:
+ return [_recursive_replace_stream_by_content(value)
+ for value in tree]
+
+
+@ajaxfunc(output_type='json')
+def fileupload(self):
+ """ Return a json copy of the web request formin which uploaded files
+ are read and their content substitute the received streams.
+ """
+ try:
+ result_dict = {}
+ for key, value in self._cw.form.iteritems():
+ result_dict[key] = _recursive_replace_stream_by_content(value)
+ return result_dict
+ except Exception, ex:
+ import traceback as tb
+ tb.print_exc(ex)
+
orig_handle = CubicWebPublisher.main_handle_request
CubicWebPublisher.main_handle_request = auto_login_handle_request
--- a/web/test/unittest_web.py Tue Jul 09 15:58:26 2013 +0200
+++ b/web/test/unittest_web.py Mon Jun 24 19:00:40 2013 +0200
@@ -16,7 +16,17 @@
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+from json import loads
+from os.path import join
+
+try:
+ import requests
+ assert [int(n) for n in requests.__version__.split('.', 2)][:2] >= [1, 2]
+except (ImportError, AssertionError):
+ requests = None
+
from logilab.common.testlib import TestCase, unittest_main
+from cubicweb.devtools.httptest import CubicWebServerTC
from cubicweb.devtools.fake import FakeRequest
class AjaxReplaceUrlTC(TestCase):
@@ -43,5 +53,45 @@
(cbname, qs, req.pageid),
req.html_headers.post_inlined_scripts[0])
+
+class FileUploadTC(CubicWebServerTC):
+
+ def setUp(self):
+ "Skip whole test class if a suitable requests module is not available"
+ if requests is None:
+ self.skipTest('Python ``requests`` module is not available')
+ super(FileUploadTC, self).setUp()
+
+ @property
+ def _post_url(self):
+ return self.request().build_url('ajax', fname='fileupload')
+
+ def _fobject(self, fname):
+ return open(join(self.datadir, fname), 'rb')
+
+ def _fcontent(self, fname):
+ return self._fobject(fname).read()
+
+ def test_single_file_upload(self):
+ files = {'file': ('schema.py', self._fobject('schema.py'))}
+ webreq = requests.post(self._post_url, files=files)
+ # check backward compat : a single uploaded file leads to a single
+ # 2-uple in the request form
+ expect = {'fname': u'fileupload',
+ 'file': ['schema.py', self._fcontent('schema.py')]}
+ self.assertEqual(webreq.status_code, 200)
+ self.assertDictEqual(expect, loads(webreq.content))
+
+ def test_multiple_file_upload(self):
+ files = [('files', ('schema.py', self._fobject('schema.py'))),
+ ('files', ('views.py', self._fobject('views.py')))]
+ webreq = requests.post(self._post_url, files=files,)
+ expect = {'fname': u'fileupload',
+ 'files': [['schema.py', self._fcontent('schema.py')],
+ ['views.py', self._fcontent('views.py')]],}
+ self.assertEqual(webreq.status_code, 200)
+ self.assertDictEqual(expect, loads(webreq.content))
+
+
if __name__ == '__main__':
unittest_main()