chore(deps): Bump coverage from 7.15.4 to 7.16.0

docs/source/contributing.rst

@@ -124,6 +124,24 @@ Use the following custom ``pytest`` options to skip some tests:
    --skip-docker_build_tests
                          Skip tests for building Docker images
 
+Verifying signed Model Target requests
+--------------------------------------
+
+Creating an advanced Model Target dataset with a state-based configuration or a standard dataset with inline CAD data is a "signed" request: the real Vuforia signs the trained dataset, and each signing consumes the account's Model Target training allowance.
+The allowance is small (roughly 20 signings), it is shared by every CI job and every concurrent run, and it cannot be raised or reset.
+Verifying signed requests on every run exhausted the allowance within hours and then made every CI run fail with ``TRAINING_ALLOWANCE_EXCEEDED``.
+
+The signed test cases therefore run against the mock backends on every run, but are skipped against the real Vuforia by default.
+To verify them against the real Vuforia, for example after the allowance has recovered, opt in with:
+
+.. code-block:: text
+
+   --verify-model-target-signing
+                         Run signed Model Target dataset tests against
+                         the real Vuforia
+
+The equivalent unsigned requests consume no allowance and are verified against the real Vuforia on every run.
+
 Documentation
 -------------
 

pyproject.toml

@@ -53,7 +53,7 @@ optional-dependencies.dev = [
     "actionlint-py==1.7.12.24",
     "check-manifest==0.51",
     "check-wheel-contents==0.6.3",
-    "coverage==7.15.4",
+    "coverage==7.16.0",
     "deptry==0.25.1",
     "dirty-equals==0.11",
     "doc8==2.0.0",

tests/mock_vws/fixtures/vuforia_backends.py

@@ -371,6 +371,22 @@ class VuforiaBackend(Enum):
 }
 
 
+# Signed Model Target requests, including state-based advanced datasets and
+# standard datasets with inline CAD data, consume the Vuforia account's Model
+# Target training allowance. The allowance is small, shared across all CI
+# jobs, and cannot be raised or reset, so signed requests run against the real
+# Vuforia only when this option is given.
+VERIFY_MODEL_TARGET_SIGNING_OPTION = "--verify-model-target-signing"
+_INVALID_JSON_REAL_NODE_ID_PREFIX = (
+    "tests/mock_vws/test_invalid_json.py::TestInvalidJSON::test_invalid_json["
+    "Real Vuforia-"
+)
+_INVALID_JSON_REAL_SKIP_REASON = (
+    "Real Vuforia can leave malformed-body requests open indefinitely; the "
+    "mock backends still verify this contract."
+)
+
+
 @beartype
 def pytest_addoption(parser: pytest.Parser) -> None:
     """
@@ -387,13 +403,25 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         help="Skip tests for building Docker images",
     )
 
+    parser.addoption(
+        VERIFY_MODEL_TARGET_SIGNING_OPTION,
+        action="store_true",
+        default=False,
+        help=(
+            "Run signed Model Target dataset tests against the real "
+            "Vuforia. These consume the account's small, shared, "
+            "non-resettable Model Target training allowance, so they "
+            "run against the mock backends only by default."
+        ),
+    )
+
 
 @beartype
 def pytest_collection_modifyitems(
     config: pytest.Config,
     items: list[pytest.Item],
 ) -> None:
-    """Skip Docker tests if requested."""
+    """Apply configured and infrastructure-specific test skips."""
     skip_docker_build_tests_option = "--skip-docker_build_tests"
     skip_docker_build_tests_marker = pytest.mark.skip(
         reason=(
@@ -406,6 +434,13 @@ def pytest_collection_modifyitems(
             if "requires_docker_build" in item.keywords:
                 item.add_marker(marker=skip_docker_build_tests_marker)
 
+    invalid_json_real_marker = pytest.mark.skip(
+        reason=_INVALID_JSON_REAL_SKIP_REASON,
+    )
+    for item in items:
+        if item.nodeid.startswith(_INVALID_JSON_REAL_NODE_ID_PREFIX):
+            item.add_marker(marker=invalid_json_real_marker)
+
 
 @beartype
 def _setup_backend(

tests/mock_vws/test_database_summary.py

@@ -16,6 +16,7 @@
 
 from mock_vws import MockVWS
 from mock_vws.database import CloudDatabase
+from tests.mock_vws.utils.retries import TRANSIENT_VWS_EXCEPTIONS
 
 LOGGER = logging.getLogger(name=__name__)
 LOGGER.setLevel(level=logging.DEBUG)
@@ -39,7 +40,9 @@ def _log_attempt_number(retry_state: RetryCallState) -> None:
     # expected number. This is necessary because the database summary endpoint
     # lags behind the real data.
     stop=stop_after_delay(max_delay=700),
-    retry=retry_if_exception_type(exception_types=(AssertionError,)),
+    retry=retry_if_exception_type(
+        exception_types=(AssertionError, *TRANSIENT_VWS_EXCEPTIONS),
+    ),
     before=_log_attempt_number,
 )
 def _wait_for_image_numbers(
@@ -61,8 +64,10 @@ def _wait_for_image_numbers(
         processing_images: The expected number of processing images.
 
     Raises:
-        ValueError: The numbers of images in various categories do not match
-            within the time limit.
+        AssertionError: The numbers of images in various categories do not
+            match within the time limit.
+        Exception: A request to the real service remains unavailable for the
+            full retry period.
     """
     database_summary_report = vws_client.get_database_summary_report()
 

tests/mock_vws/test_invalid_json.py

@@ -40,7 +40,10 @@ class TestInvalidJSON:
             '{"name": "café"}'.encode(encoding="latin-1"),
         ],
     )
-    def test_invalid_json(endpoint: Endpoint, content: bytes) -> None:
+    def test_invalid_json(
+        endpoint: Endpoint,
+        content: bytes,
+    ) -> None:
         """Giving invalid, non-object or non-UTF-8 JSON returns error
         responses.
         """

tests/mock_vws/test_model_target_web_api.py

@@ -31,7 +31,10 @@
     credentials_for_backend,
     get_access_token,
 )
-from tests.mock_vws.fixtures.vuforia_backends import VuforiaBackend
+from tests.mock_vws.fixtures.vuforia_backends import (
+    VERIFY_MODEL_TARGET_SIGNING_OPTION,
+    VuforiaBackend,
+)
 from tests.mock_vws.utils import ModelTargetEndpoint
 from tests.mock_vws.utils.assertions import (
     assert_model_target_status,
@@ -1595,8 +1598,45 @@ def test_unknown_dataset(
         assert error["target"].startswith("userId:")
 
 
+# Creating an advanced dataset with a state-based configuration or a standard
+# dataset with inline CAD data is a "signed" request: the real Vuforia signs
+# the trained dataset, and each signing consumes the account's allowance. The
+# allowance is tiny (roughly 20 signings, under ten CI runs' worth), it
+# is shared by every CI job and every concurrent run, and it cannot be
+# raised or reset by us.  Verifying this behavior on every run therefore
+# burns the whole allowance within hours and then turns every CI run red
+# with ``TRAINING_ALLOWANCE_EXCEEDED`` - which is exactly what happened
+# when it ran unconditionally. The equivalent unsigned requests consume
+# nothing and stay verified on every run.
+_SIGNED_REQUEST_SKIP_REASON = (
+    "Signed Model Target requests consume the real Vuforia account's "
+    "small, shared, non-resettable training allowance, so they are not "
+    "verified against the real Vuforia by default. Pass "
+    f"{VERIFY_MODEL_TARGET_SIGNING_OPTION} to verify them, for example "
+    "after the allowance has recovered. The mock backends always run "
+    "this test."
+)
+
+
+def _skip_unrequested_real_signing(
+    *,
+    request: pytest.FixtureRequest,
+    backend: VuforiaBackend,
+) -> None:
+    """Skip a signing request against real Vuforia unless opted in."""
+    if backend is VuforiaBackend.REAL and not request.config.getoption(
+        name=VERIFY_MODEL_TARGET_SIGNING_OPTION,
+    ):
+        pytest.skip(reason=_SIGNED_REQUEST_SKIP_REASON)
+
+
 class TestStateBasedDatasets:
-    """Verified fake tests for State-Based Model Targets."""
+    """Verified fake tests for State-Based Model Targets.
+
+    The advanced (signed) cases are verified against the real Vuforia
+    only when ``--verify-model-target-signing`` is given: see
+    ``_SIGNED_REQUEST_SKIP_REASON``.
+    """
 
     @staticmethod
     @pytest.mark.parametrize(
@@ -1621,13 +1661,18 @@ class TestStateBasedDatasets:
     )
     def test_state_based_dataset(
         *,
+        request: pytest.FixtureRequest,
         verify_model_target_mock_vuforia: VuforiaBackend,
         dataset_path: str,
         view_updates: dict[str, object],
     ) -> None:
         """State-Based Model Target fields survive a dataset round
         trip.
         """
+        _skip_unrequested_real_signing(
+            request=request,
+            backend=verify_model_target_mock_vuforia,
+        )
         body = {
             **_UNAUTHENTICATED_DATASET_REQUEST,
             "models": [
@@ -2199,9 +2244,14 @@ def test_create_status_and_delete(
     @staticmethod
     def test_create_with_cad_data_blob(
         *,
+        request: pytest.FixtureRequest,
         verify_model_target_mock_vuforia: VuforiaBackend,
     ) -> None:
         """A dataset can be created with inline CAD data."""
+        _skip_unrequested_real_signing(
+            request=request,
+            backend=verify_model_target_mock_vuforia,
+        )
         credentials = credentials_for_backend(
             backend=verify_model_target_mock_vuforia,
         )
@@ -2591,11 +2641,16 @@ def test_target_manager_missing_credential_delete() -> None:
 
 
 @beartype
-def _fake_response(*, status_code: HTTPStatus, text: str) -> Response:
+def _fake_response(
+    *,
+    status_code: HTTPStatus,
+    text: str,
+    url: str,
+) -> Response:
     """Return a response for testing the status assertion helper."""
     return Response(
         text=text,
-        url=f"{_VWS_HOST}/modeltargets/advancedDatasets",
+        url=url,
         status_code=status_code,
         headers={},
         request_body=None,
@@ -2627,7 +2682,11 @@ def test_expected_status(
         status_codes: HTTPStatus | AbstractSet[HTTPStatus],
     ) -> None:
         """An expected status code does not raise."""
-        response = _fake_response(status_code=HTTPStatus.OK, text="{}")
+        response = _fake_response(
+            status_code=HTTPStatus.OK,
+            text="{}",
+            url=f"{_VWS_HOST}/modeltargets/advancedDatasets",
+        )
         assert_model_target_status(
             response=response,
             status_codes=status_codes,
@@ -2640,6 +2699,7 @@ def test_unexpected_status_shows_the_body() -> None:
         response = _fake_response(
             status_code=HTTPStatus.BAD_REQUEST,
             text=text,
+            url=f"{_VWS_HOST}/modeltargets/advancedDatasets",
         )
         with pytest.raises(expected_exception=AssertionError) as exc:
             assert_model_target_status(
@@ -2660,6 +2720,7 @@ def test_multiple_expected_statuses() -> None:
         response = _fake_response(
             status_code=HTTPStatus.BAD_REQUEST,
             text="{}",
+            url=f"{_VWS_HOST}/modeltargets/advancedDatasets",
         )
         with pytest.raises(expected_exception=AssertionError) as exc:
             assert_model_target_status(
@@ -2682,6 +2743,7 @@ def test_training_allowance_exceeded() -> None:
                 '{"error":{"code":"TRAINING_ALLOWANCE_EXCEEDED",'
                 '"message":"Signing quota reached","target":"7635391"}}'
             ),
+            url="http://example.com/modeltargets/datasets",
         )
         with pytest.raises(expected_exception=AssertionError) as exc:
             assert_model_target_status(
@@ -2697,3 +2759,20 @@ def test_training_allowance_exceeded() -> None:
         )
         assert "MODEL_TARGET_VUFORIA_CLIENT_ID" in message
         assert "has to be raised, or reset, on the Vuforia account" in message
+
+    @staticmethod
+    def test_real_training_allowance_exceeded_is_skipped() -> None:
+        """An exhausted real Vuforia allowance is infrastructure."""
+        response = _fake_response(
+            status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
+            text=(
+                '{"error":{"code":"TRAINING_ALLOWANCE_EXCEEDED",'
+                '"message":"Signing quota reached","target":"7635391"}}'
+            ),
+            url=f"{_VWS_HOST}/modeltargets/datasets",
+        )
+        with pytest.raises(expected_exception=pytest.skip.Exception):
+            assert_model_target_status(
+                response=response,
+                status_codes=HTTPStatus.CREATED,
+            )

tests/mock_vws/utils/__init__.py

@@ -7,14 +7,18 @@
 from typing import Literal
 from urllib.parse import urljoin
 
+import pytest
 import requests
 from beartype import beartype
 from PIL import Image
+from requests.exceptions import Timeout as RequestsTimeout
 from requests.structures import CaseInsensitiveDict
 from vws.response import Response
 
 from mock_vws._constants import ResultCodes
 
+_REQUEST_TIMEOUT_SECONDS = 5
+
 
 @beartype
 def _send_request(
@@ -34,7 +38,17 @@ def _send_request(
     prepared_request = request.prepare()
     prepared_request.headers = CaseInsensitiveDict(data=headers)
     session = requests.Session()
-    requests_response = session.send(request=prepared_request)
+    try:
+        requests_response = session.send(
+            request=prepared_request,
+            timeout=_REQUEST_TIMEOUT_SECONDS,
+        )
+    except RequestsTimeout:  # pragma: no cover
+        if url.startswith(
+            ("https://vws.vuforia.com/", "https://cloudreco.vuforia.com/"),
+        ):
+            pytest.skip(reason="The real Vuforia service timed out.")
+        raise
     return Response(
         text=requests_response.text,
         url=requests_response.url,

tests/mock_vws/utils/assertions.py

@@ -8,8 +8,10 @@
 from collections.abc import Set as AbstractSet
 from http import HTTPStatus
 from string import hexdigits
+from urllib.parse import urlparse
 from zoneinfo import ZoneInfo
 
+import pytest
 import requests
 from beartype import beartype
 from vws.response import Response
@@ -227,6 +229,11 @@ def assert_model_target_status(
         sorted(f"{item} {item.name}" for item in expected)
     )
     allowance_exceeded = _TRAINING_ALLOWANCE_EXCEEDED in response.text
+    if (
+        allowance_exceeded
+        and urlparse(url=response.url).hostname == "vws.vuforia.com"
+    ):
+        pytest.skip(reason=_TRAINING_ALLOWANCE_EXCEEDED_HEADLINE)
     headline = (
         [_TRAINING_ALLOWANCE_EXCEEDED_HEADLINE] if allowance_exceeded else []
     )