Skip signed Model Target requests against real Vuforia by default

Creating an advanced Model Target dataset with a state-based
configuration is a signed request which consumes the Vuforia account's
small, shared, non-resettable training allowance. Running these cases
against the real Vuforia on every CI run exhausted the allowance and
turned every run red with TRAINING_ALLOWANCE_EXCEEDED.

Skip them against the real Vuforia by default, add a
--verify-model-target-signing option to opt back in, and document why.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: adamtheturtle

docs/source/contributing.rst

@@ -124,6 +124,24 @@ Use the following custom ``pytest`` options to skip some tests:
    --skip-docker_build_tests
                          Skip tests for building Docker images
 
+Verifying signed Model Target requests
+--------------------------------------
+
+Creating an advanced Model Target dataset with a state-based configuration is a "signed" request: the real Vuforia signs the trained dataset, and each signing consumes the account's Model Target training allowance.
+The allowance is small (roughly 20 signings), it is shared by every CI job and every concurrent run, and it cannot be raised or reset.
+Verifying signed requests on every run exhausted the allowance within hours and then made every CI run fail with ``TRAINING_ALLOWANCE_EXCEEDED``.
+
+The signed test cases therefore run against the mock backends on every run, but are skipped against the real Vuforia by default.
+To verify them against the real Vuforia, for example after the allowance has recovered, opt in with:
+
+.. code-block:: text
+
+   --verify-model-target-signing
+                         Run signed Model Target dataset tests against
+                         the real Vuforia
+
+The equivalent unsigned requests (a standard dataset, or an advanced dataset without a state-based configuration) consume no allowance and are verified against the real Vuforia on every run.
+
 Documentation
 -------------
 

tests/mock_vws/fixtures/vuforia_backends.py

@@ -371,6 +371,14 @@ class VuforiaBackend(Enum):
 }
 
 
+# Signed Model Target requests (advanced datasets with a state-based
+# configuration) consume the Vuforia account's Model Target training
+# allowance.  The allowance is small, shared across all CI jobs, and
+# cannot be raised or reset, so signed requests run against the real
+# Vuforia only when this option is given.
+VERIFY_MODEL_TARGET_SIGNING_OPTION = "--verify-model-target-signing"
+
+
 @beartype
 def pytest_addoption(parser: pytest.Parser) -> None:
     """
@@ -387,6 +395,18 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         help="Skip tests for building Docker images",
     )
 
+    parser.addoption(
+        VERIFY_MODEL_TARGET_SIGNING_OPTION,
+        action="store_true",
+        default=False,
+        help=(
+            "Run signed Model Target dataset tests against the real "
+            "Vuforia. These consume the account's small, shared, "
+            "non-resettable Model Target training allowance, so they "
+            "run against the mock backends only by default."
+        ),
+    )
+
 
 @beartype
 def pytest_collection_modifyitems(

tests/mock_vws/test_model_target_web_api.py

@@ -31,7 +31,10 @@
     credentials_for_backend,
     get_access_token,
 )
-from tests.mock_vws.fixtures.vuforia_backends import VuforiaBackend
+from tests.mock_vws.fixtures.vuforia_backends import (
+    VERIFY_MODEL_TARGET_SIGNING_OPTION,
+    VuforiaBackend,
+)
 from tests.mock_vws.utils import ModelTargetEndpoint
 from tests.mock_vws.utils.assertions import (
     assert_model_target_status,
@@ -1595,8 +1598,34 @@ def test_unknown_dataset(
         assert error["target"].startswith("userId:")
 
 
+# Creating an advanced dataset with a state-based configuration is a
+# "signed" request: the real Vuforia signs the trained dataset, and each
+# signing consumes the account's Model Target training allowance.  The
+# allowance is tiny (roughly 20 signings, under ten CI runs' worth), it
+# is shared by every CI job and every concurrent run, and it cannot be
+# raised or reset by us.  Verifying this behavior on every run therefore
+# burns the whole allowance within hours and then turns every CI run red
+# with ``TRAINING_ALLOWANCE_EXCEEDED`` - which is exactly what happened
+# when it ran unconditionally.  The equivalent unsigned requests (a
+# standard dataset, or an advanced dataset without a state-based
+# configuration) consume nothing and stay verified on every run.
+_SIGNED_REQUEST_SKIP_REASON = (
+    "Signed Model Target requests consume the real Vuforia account's "
+    "small, shared, non-resettable training allowance, so they are not "
+    "verified against the real Vuforia by default. Pass "
+    f"{VERIFY_MODEL_TARGET_SIGNING_OPTION} to verify them, for example "
+    "after the allowance has recovered. The mock backends always run "
+    "this test."
+)
+
+
 class TestStateBasedDatasets:
-    """Verified fake tests for State-Based Model Targets."""
+    """Verified fake tests for State-Based Model Targets.
+
+    The advanced (signed) cases are verified against the real Vuforia
+    only when ``--verify-model-target-signing`` is given: see
+    ``_SIGNED_REQUEST_SKIP_REASON``.
+    """
 
     @staticmethod
     @pytest.mark.parametrize(
@@ -1621,13 +1650,22 @@ class TestStateBasedDatasets:
     )
     def test_state_based_dataset(
         *,
+        request: pytest.FixtureRequest,
         verify_model_target_mock_vuforia: VuforiaBackend,
         dataset_path: str,
         view_updates: dict[str, object],
     ) -> None:
         """State-Based Model Target fields survive a dataset round
         trip.
         """
+        if (
+            verify_model_target_mock_vuforia is VuforiaBackend.REAL
+            and dataset_path == "/modeltargets/advancedDatasets"
+            and not request.config.getoption(
+                name=VERIFY_MODEL_TARGET_SIGNING_OPTION,
+            )
+        ):
+            pytest.skip(reason=_SIGNED_REQUEST_SKIP_REASON)
         body = {
             **_UNAUTHENTICATED_DATASET_REQUEST,
             "models": [