cms_form: add :file marshaller

Rework file processing so that files come ready from request data.

cms_form/marshallers.py

@@ -1,26 +1,25 @@
 # Copyright 2018 Simone Orsi
 # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html).
 
+import base64
 import html
 from collections.abc import Callable
 from dataclasses import dataclass
 from typing import Any
 
+import werkzeug
+
+from odoo.tools import pycompat
+from odoo.tools.mimetypes import guess_mimetype
+
 from . import utils
 
 
 def marshal_request_values(values):
     """Transform given request values using marshallers.
 
-    Available marshallers:
-
-    * `:int` transform to integer
-    * `:float` transform to float
-    * `:list` transform to list of values
-    * `:dict` transform to dictionary of values
+    Available marshallers: see Marshaller class.
     """
-    # TODO: add docs
-    # TODO: support combinations like `:list:int` or `:dict:int`
     return Marshaller(values).marshall()
 
 
@@ -57,13 +56,16 @@ def _collect_todo(self):
                 done.add(k)
 
     def _marshallers(self):
+        # TODO: add docs
+        # TODO: support combinations like `:list:int` or `:dict:int`
         return (
             (":esc", self.marshal_esc),
             (":dict:list", self.marshal_dict_list),
             (":list", self.marshal_list),
             (":dict", self.marshal_dict),
             (":int", self.marshal_int),
             (":float", self.marshal_float),
+            (":file", self.marshal_file),
         )
 
     def marshall(self):
@@ -198,3 +200,35 @@ def parse_key(key):
                 item[inner_key] = value
             res.append(item)
         return main_key, res
+
+    def marshal_file(self, orig_key, orig_value):
+        k = orig_key[: -len(":file")]
+        value = orig_value
+        if isinstance(value, werkzeug.datastructures.FileStorage):
+            _value = self._filedata_from_filestorage(value)
+        else:
+            mimetype = guess_mimetype(value)
+            _value = {
+                "value": value,
+                "raw_value": value,
+                "mimetype": mimetype,
+                "content_type": mimetype,
+            }
+        _value["_from_request"] = True
+        return k, _value
+
+    def _filedata_from_filestorage(self, fs):
+        raw_value = fs.read()
+        value = base64.b64encode(raw_value)
+        value = pycompat.to_text(value)
+        data = dict(raw_value=raw_value, value=value)
+        for attr in (
+            "content_length",
+            "content_type",
+            "filename",
+            "headers",
+            "mimetype",
+            "mimetype_params",
+        ):
+            data[attr] = getattr(fs, attr)
+        return data

cms_form/models/cms_form_mixin.py

@@ -502,7 +502,25 @@ def form_get_request_values(self):
         # normal fields
         res = marshallers.marshal_request_values(_values)
         # file fields
-        res.update({k: v for k, v in self.request.files.items()})
+        files = self.request.files
+        # Convert files always. Main reason:
+        # * file descriptors will be consumed on 1st read
+        # * homegenous handling of files
+        # * no need to parse metadata down the stack as is done by the marshaller
+        parsed_files = getattr(self.request, "_cms_form_files_processed", None)
+        if files and not parsed_files:
+            _file_values = {}
+            _file_fields = self.form_file_fields()
+            for fname, fobj in files.items():
+                if fname in _file_fields:
+                    # fake field name enforcing marshaller
+                    if not fname.endswith(":file"):
+                        fname = f"{fname}:file"
+                    _file_values[fname] = fobj
+            file_values = marshallers.marshal_request_values(_file_values)
+            self.request._cms_form_files_processed = file_values
+        elif parsed_files:
+            res.update(parsed_files)
         return res
 
     # TODO: rename to form_load

cms_form/models/widgets/widget_binary.py

@@ -7,7 +7,7 @@
 
 from odoo import fields, models
 from odoo.tools import pycompat
-from odoo.tools.mimetypes import guess_mimetype
+from odoo.tools.image import image_data_uri
 
 
 class BinaryWidget(models.AbstractModel):
@@ -19,29 +19,6 @@ def w_load(self, **req_values):
         value = super().w_load(**req_values)
         return self.binary_to_form(value, **req_values)
 
-    def binary_to_form(self, value, **req_values):
-        _value = None
-        from_request = False
-        if value:
-            if isinstance(value, werkzeug.datastructures.FileStorage):
-                from_request = True
-                byte_content = value.read()
-                value = base64.b64encode(byte_content)
-                value = pycompat.to_text(value)
-            else:
-                value = pycompat.to_text(value)
-                byte_content = base64.b64decode(value)
-            mimetype = guess_mimetype(byte_content)
-            _value = {
-                "value": value,
-                "raw_value": value,
-                "mimetype": mimetype,
-                "from_request": from_request,
-            }
-            if mimetype.startswith("image/"):
-                _value["value"] = "data:{};base64,{}".format(mimetype, value)
-        return _value
-
     def w_extract(self, **req_values):
         value = super().w_extract(**req_values)
         return self.form_to_binary(value, **req_values)
@@ -92,6 +69,12 @@ class ImageWidget(models.AbstractModel):
 
     w_template = fields.Char(default="cms_form.field_widget_image")
 
+    def binary_to_form(self, value, **req_values):
+        _value = super().binary_to_form(value, **req_values)
+        if _value and value.get("mimetype", "").startswith("image/"):
+            _value["value"] = image_data_uri(_value["value"])
+        return _value
+
 
 class FileWidget(models.AbstractModel):
     _name = "cms.form.widget.binary"

cms_form/tests/test_marshallers.py

@@ -3,9 +3,10 @@
 
 import unittest
 
-from werkzeug.datastructures import MultiDict
+from werkzeug.datastructures import FileMultiDict, Headers, MultiDict
 
 from .. import marshallers
+from .utils import fake_file_from_request, file_as_stream
 
 
 class TestMarshallers(unittest.TestCase):
@@ -111,3 +112,36 @@ def test_marshal_dict_list(self):
         self.assertEqual(marshalled["c"], "3")
         self.assertNotIn("b.1.x:dict:list", marshalled)
         self.assertNotIn("b.2.y:dict:list", marshalled)
+
+    def test_marshal_file(self):
+        data = FileMultiDict()
+        content = b"a,b,c\n1,2,3\n4,5,6"
+        with file_as_stream(content) as stream:
+            data.add_file(
+                "one:file",
+                fake_file_from_request(
+                    "one",
+                    stream=stream,
+                    filename="one.csv",
+                    content_type="text/csv",
+                    content_length=len(content),
+                ),
+            )
+            marshalled = marshallers.marshal_request_values(data)
+        self.assertEqual(
+            marshalled["one"],
+            {
+                "_from_request": True,
+                "content_length": 17,
+                "content_type": "text/csv",
+                "filename": "one.csv",
+                "headers": Headers(
+                    [("Content-Type", "text/csv"), ("Content-Length", "17")]
+                ),
+                "mimetype": "text/csv",
+                "mimetype_params": {},
+                "raw_value": b"a,b,c\n1,2,3\n4,5,6",
+                "value": "YSxiLGMKMSwyLDMKNCw1LDY=",
+            },
+        )
+        self.assertNotIn("one:file", marshalled)

cms_form/tests/utils.py

@@ -93,13 +93,19 @@ def fake_session(env, **kw):
 
 
 @contextmanager
-def b64_as_stream(b64_content):
+def file_as_stream(content):
     stream = io.BytesIO()
-    stream.write(base64.b64decode(b64_content))
+    stream.write(content)
     stream.seek(0)
     yield stream
     stream.close()
 
 
+@contextmanager
+def b64_as_stream(b64_content):
+    with file_as_stream(base64.b64decode(b64_content)) as stream:
+        yield stream
+
+
 def fake_file_from_request(input_name, stream, **kw):
     return FileStorage(name=input_name, stream=stream, **kw)