Merge pull request #3505 from VWS-Python/adamtheturtle/issue-3392

Respond to non-UTF-8 and truncated multipart bodies (#3392)

Author: adamtheturtle

.github/workflows/test.yml

@@ -46,6 +46,7 @@ jobs:
           - tests/mock_vws/test_query.py::TestTargetStatusFailed
           - tests/mock_vws/test_query.py::TestDateFormats
           - tests/mock_vws/test_query.py::TestInactiveProject
+          - tests/mock_vws/test_query.py::TestTruncatedBody
           - tests/mock_vws/test_add_target.py::TestContentTypes
           - tests/mock_vws/test_add_target.py::TestMissingData
           - tests/mock_vws/test_add_target.py::TestWidth

newsfragments/3392.change

@@ -0,0 +1 @@
+Respond, rather than failing to respond, to a request body which is not UTF-8 and to a query request whose ``multipart/form-data`` body ends before its closing boundary.

src/mock_vws/_query_tools.py

@@ -1,20 +1,18 @@
 """Tools for making Vuforia queries."""
 
 import base64
-import io
 import uuid
 from collections.abc import Iterable, Mapping
-from email.message import EmailMessage
 from typing import Any
 
 from beartype import beartype
-from werkzeug.formparser import MultiPartParser
 
 from mock_vws._base64_decoding import decode_base64
 from mock_vws._constants import ResultCodes, TargetStatuses
 from mock_vws._database_matchers import get_database_matching_client_keys
 from mock_vws._matching import matching_targets
 from mock_vws._mock_common import json_dump
+from mock_vws._query_validators.multipart import parse_multipart
 from mock_vws.database import CloudDatabase
 from mock_vws.image_matchers import ImageMatcher
 
@@ -42,15 +40,9 @@ def get_query_match_response_text(
     Returns:
         The response text for a query endpoint request.
     """
-    email_message = EmailMessage()
-    email_message["Content-Type"] = request_headers["Content-Type"]
-    boundary = email_message.get_boundary(failobj="")
-
-    parser = MultiPartParser()
-    fields, files = parser.parse(
-        stream=io.BytesIO(initial_bytes=request_body),
-        boundary=boundary.encode(encoding="utf-8"),
-        content_length=len(request_body),
+    fields, files = parse_multipart(
+        request_headers=request_headers,
+        request_body=request_body,
     )
 
     max_num_results = fields.get(key="max_num_results", default="1")

src/mock_vws/_query_validators/exceptions.py

@@ -201,6 +201,29 @@ def __init__(self) -> None:
         }
 
 
+@beartype
+class NoContentDispositionError(ValidatorError):
+    """Exception raised when a part of a multipart body has no
+    ``Content-Disposition`` header.
+    """
+
+    def __init__(self) -> None:
+        """Initialize a missing ``Content-Disposition`` header
+        response.
+        """
+        super().__init__()
+        self.status_code = HTTPStatus.BAD_REQUEST
+        self.response_text = (
+            "Could find no Content-Disposition header within part"
+        )
+        self.headers = {
+            **_BASE_HEADERS,
+            "Content-Type": "text/plain;charset=utf-8",
+            "Date": http_date(),
+            "Content-Length": str(object=len(self.response_text)),
+        }
+
+
 @beartype
 class AuthHeaderMissingError(ValidatorError):
     """Exception raised when an auth header is not given."""

src/mock_vws/_query_validators/fields_validators.py

@@ -1,14 +1,12 @@
 """Validators for the fields given."""
 
-import io
 import logging
 from collections.abc import Mapping
-from email.message import EmailMessage
 
 from beartype import beartype
-from werkzeug.formparser import MultiPartParser
 
 from mock_vws._query_validators.exceptions import UnknownParametersError
+from mock_vws._query_validators.multipart import parse_multipart
 
 _LOGGER = logging.getLogger(name=__name__)
 
@@ -27,15 +25,12 @@ def validate_extra_fields(
 
     Raises:
         UnknownParametersError: Extra fields are given.
+        NoContentDispositionError: A part of the body has no
+            ``Content-Disposition`` header.
     """
-    email_message = EmailMessage()
-    email_message["Content-Type"] = request_headers["Content-Type"]
-    boundary = email_message.get_boundary(failobj="")
-    parser = MultiPartParser()
-    fields, files = parser.parse(
-        stream=io.BytesIO(initial_bytes=request_body),
-        boundary=boundary.encode(encoding="utf-8"),
-        content_length=len(request_body),
+    fields, files = parse_multipart(
+        request_headers=request_headers,
+        request_body=request_body,
     )
     parsed_keys = fields.keys() | files.keys()
     known_parameters = {"image", "max_num_results", "include_target_data"}

src/mock_vws/_query_validators/image_validators.py

@@ -3,18 +3,17 @@
 import io
 import logging
 from collections.abc import Mapping
-from email.message import EmailMessage
 
 from beartype import beartype
 from werkzeug.datastructures import FileStorage, MultiDict
-from werkzeug.formparser import MultiPartParser
 
 from mock_vws._image_opening import open_image
 from mock_vws._query_validators.exceptions import (
     BadImageError,
     ImageNotGivenError,
     RequestEntityTooLargeError,
 )
+from mock_vws._query_validators.multipart import parse_multipart
 
 _LOGGER = logging.getLogger(name=__name__)
 
@@ -34,14 +33,9 @@ def _parse_multipart_files(
     Returns:
         The files parsed from the multipart body.
     """
-    email_message = EmailMessage()
-    email_message["Content-Type"] = request_headers["Content-Type"]
-    boundary = email_message.get_boundary(failobj="")
-    parser = MultiPartParser()
-    _, files = parser.parse(
-        stream=io.BytesIO(initial_bytes=request_body),
-        boundary=boundary.encode(encoding="utf-8"),
-        content_length=len(request_body),
+    _, files = parse_multipart(
+        request_headers=request_headers,
+        request_body=request_body,
     )
     return files
 

src/mock_vws/_query_validators/include_target_data_validators.py

@@ -1,14 +1,12 @@
 """Validators for the ``include_target_data`` field."""
 
-import io
 import logging
 from collections.abc import Mapping
-from email.message import EmailMessage
 
 from beartype import beartype
-from werkzeug.formparser import MultiPartParser
 
 from mock_vws._query_validators.exceptions import InvalidIncludeTargetDataError
+from mock_vws._query_validators.multipart import parse_multipart
 
 _LOGGER = logging.getLogger(name=__name__)
 
@@ -31,14 +29,9 @@ def validate_include_target_data(
         InvalidIncludeTargetDataError: The ``include_target_data`` field is not
             an accepted value.
     """
-    email_message = EmailMessage()
-    email_message["Content-Type"] = request_headers["Content-Type"]
-    boundary = email_message.get_boundary(failobj="")
-    parser = MultiPartParser()
-    fields, _ = parser.parse(
-        stream=io.BytesIO(initial_bytes=request_body),
-        boundary=boundary.encode(encoding="utf-8"),
-        content_length=len(request_body),
+    fields, _ = parse_multipart(
+        request_headers=request_headers,
+        request_body=request_body,
     )
     include_target_data = fields.get(key="include_target_data", default="top")
     allowed_included_target_data = {"top", "all", "none"}

src/mock_vws/_query_validators/multipart.py

@@ -0,0 +1,109 @@
+"""Parsing of the ``multipart/form-data`` bodies given to the query
+API.
+"""
+
+import io
+import logging
+from collections.abc import Mapping
+from email.message import EmailMessage
+
+from beartype import beartype
+from werkzeug.datastructures import FileStorage, MultiDict
+from werkzeug.formparser import MultiPartParser
+
+from mock_vws._query_validators.exceptions import NoContentDispositionError
+
+_LOGGER = logging.getLogger(name=__name__)
+
+
+@beartype
+def _parse_with_boundary(
+    *,
+    request_body: bytes,
+    boundary: bytes,
+) -> tuple[MultiDict[str, str], MultiDict[str, FileStorage]]:
+    """Parse a multipart body, requiring it to be complete.
+
+    Args:
+        request_body: The body of the request.
+        boundary: The multipart boundary, without its leading dashes.
+
+    Returns:
+        The fields and the files parsed from the multipart body.
+
+    Raises:
+        ValueError: The body is not a complete multipart body.
+    """
+    parser = MultiPartParser()
+    return parser.parse(
+        stream=io.BytesIO(initial_bytes=request_body),
+        boundary=boundary,
+        content_length=len(request_body),
+    )
+
+
+@beartype
+def parse_multipart(
+    *,
+    request_headers: Mapping[str, str],
+    request_body: bytes,
+) -> tuple[MultiDict[str, str], MultiDict[str, FileStorage]]:
+    """Parse the multipart body of a query request.
+
+    Vuforia accepts a body which ends before its closing boundary, as a
+    client which is cut off mid-upload sends, and treats the end of the body
+    as the end of the part being uploaded.  ``MultiPartParser`` rejects such a
+    body, so we give it the closing boundary which the client did not send.
+
+    Args:
+        request_headers: The headers sent with the request.
+        request_body: The body of the request.
+
+    Returns:
+        The fields and the files parsed from the multipart body.
+
+    Raises:
+        NoContentDispositionError: The body ends within the headers of a part,
+            or a part has no ``Content-Disposition`` header, so no part can be
+            named.
+    """
+    email_message = EmailMessage()
+    email_message["Content-Type"] = request_headers["Content-Type"]
+    boundary = email_message.get_boundary(failobj="").encode(encoding="utf-8")
+    closing_boundary = b"\r\n--" + boundary + b"--\r\n"
+
+    # A body which ends part-way through the closing boundary keeps those
+    # bytes: Vuforia gives them to the part being uploaded rather than
+    # ignoring them.  We therefore try the body as it was sent before we try
+    # it without its incomplete closing boundary.
+    without_partial_boundary = request_body
+    for length in reversed(range(1, len(closing_boundary))):
+        if request_body.endswith(closing_boundary[:length]):
+            without_partial_boundary = request_body[:-length]
+            break
+
+    candidates = (
+        request_body,
+        # The body ended within the data of a part.
+        request_body + closing_boundary,
+        # The body ended part-way through a header of a part, so the blank
+        # line which ends the headers of that part is missing too.
+        request_body + b"\r\n" + closing_boundary,
+        # The body ended part-way through the closing boundary.
+        without_partial_boundary + closing_boundary,
+    )
+
+    for candidate in candidates:
+        try:
+            return _parse_with_boundary(
+                request_body=candidate,
+                boundary=boundary,
+            )
+        except ValueError:
+            continue
+
+    # Every remaining body is one in which a part has no usable
+    # ``Content-Disposition`` header, either because the body ends before that
+    # header is complete or because the part does not have one.
+    _LOGGER.warning(msg="A part has no Content-Disposition header.")
+    raise NoContentDispositionError

src/mock_vws/_query_validators/num_results_validators.py

@@ -1,17 +1,15 @@
 """Validators for the ``max_num_results`` fields."""
 
-import io
 import logging
 from collections.abc import Mapping
-from email.message import EmailMessage
 
 from beartype import beartype
-from werkzeug.formparser import MultiPartParser
 
 from mock_vws._query_validators.exceptions import (
     InvalidMaxNumResultsError,
     MaxNumResultsOutOfRangeError,
 )
+from mock_vws._query_validators.multipart import parse_multipart
 
 _LOGGER = logging.getLogger(name=__name__)
 
@@ -36,14 +34,9 @@ def validate_max_num_results(
         MaxNumResultsOutOfRangeError: The ``max_num_results`` given is not in
             range.
     """
-    email_message = EmailMessage()
-    email_message["Content-Type"] = request_headers["Content-Type"]
-    boundary = email_message.get_boundary(failobj="")
-    parser = MultiPartParser()
-    fields, _ = parser.parse(
-        stream=io.BytesIO(initial_bytes=request_body),
-        boundary=boundary.encode(encoding="utf-8"),
-        content_length=len(request_body),
+    fields, _ = parse_multipart(
+        request_headers=request_headers,
+        request_body=request_body,
     )
     max_num_results = fields.get(key="max_num_results", default="1")
 

src/mock_vws/_services_validators/json_validators.py

@@ -52,17 +52,20 @@ def validate_json(*, request_body: bytes, request_path: str) -> None:
         request_path: The path of the request.
 
     Raises:
-        BadRequestError: The request body includes invalid JSON for the
-            VuMark instance generation endpoint.
-        FailError: The request body includes invalid JSON for other
-            endpoints.
+        BadRequestError: The request body is not valid UTF-8, or includes
+            invalid JSON, for the VuMark instance generation endpoint.
+        FailError: The request body is not valid UTF-8, or includes invalid
+            JSON, for other endpoints.
     """
     if not request_body:
         return
 
     try:
-        request_json = json.loads(s=request_body.decode())
-    except JSONDecodeError as exc:
+        # Vuforia gives the same response for a body which is not UTF-8, such
+        # as JSON encoded as latin-1, as it gives for a body which is not
+        # valid JSON.
+        request_json = json.loads(s=request_body.decode(encoding="utf-8"))
+    except (JSONDecodeError, UnicodeDecodeError) as exc:
         _LOGGER.warning(msg="The request body is not valid JSON.")
         if request_path.endswith("/instances"):
             raise BadRequestError from exc