diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 09faeb7b3..24ab6e067 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -124,6 +124,24 @@ Use the following custom ``pytest`` options to skip some tests: --skip-docker_build_tests Skip tests for building Docker images +Verifying signed Model Target requests +-------------------------------------- + +Creating an advanced Model Target dataset with a state-based configuration is a "signed" request: the real Vuforia signs the trained dataset, and each signing consumes the account's Model Target training allowance. +The allowance is small (roughly 20 signings), it is shared by every CI job and every concurrent run, and it cannot be raised or reset. +Verifying signed requests on every run exhausted the allowance within hours and then made every CI run fail with ``TRAINING_ALLOWANCE_EXCEEDED``. + +The signed test cases therefore run against the mock backends on every run, but are skipped against the real Vuforia by default. +To verify them against the real Vuforia, for example after the allowance has recovered, opt in with: + +.. code-block:: text + + --verify-model-target-signing + Run signed Model Target dataset tests against + the real Vuforia + +The equivalent unsigned requests (a standard dataset, or an advanced dataset without a state-based configuration) consume no allowance and are verified against the real Vuforia on every run. + Documentation ------------- diff --git a/tests/mock_vws/fixtures/vuforia_backends.py b/tests/mock_vws/fixtures/vuforia_backends.py index 11dae25f5..533e9dea0 100644 --- a/tests/mock_vws/fixtures/vuforia_backends.py +++ b/tests/mock_vws/fixtures/vuforia_backends.py @@ -371,6 +371,14 @@ class VuforiaBackend(Enum): } +# Signed Model Target requests (advanced datasets with a state-based +# configuration) consume the Vuforia account's Model Target training +# allowance. The allowance is small, shared across all CI jobs, and +# cannot be raised or reset, so signed requests run against the real +# Vuforia only when this option is given. +VERIFY_MODEL_TARGET_SIGNING_OPTION = "--verify-model-target-signing" + + @beartype def pytest_addoption(parser: pytest.Parser) -> None: """ @@ -387,6 +395,18 @@ def pytest_addoption(parser: pytest.Parser) -> None: help="Skip tests for building Docker images", ) + parser.addoption( + VERIFY_MODEL_TARGET_SIGNING_OPTION, + action="store_true", + default=False, + help=( + "Run signed Model Target dataset tests against the real " + "Vuforia. These consume the account's small, shared, " + "non-resettable Model Target training allowance, so they " + "run against the mock backends only by default." + ), + ) + @beartype def pytest_collection_modifyitems( diff --git a/tests/mock_vws/test_model_target_web_api.py b/tests/mock_vws/test_model_target_web_api.py index 138340abf..1e930040e 100644 --- a/tests/mock_vws/test_model_target_web_api.py +++ b/tests/mock_vws/test_model_target_web_api.py @@ -31,7 +31,10 @@ credentials_for_backend, get_access_token, ) -from tests.mock_vws.fixtures.vuforia_backends import VuforiaBackend +from tests.mock_vws.fixtures.vuforia_backends import ( + VERIFY_MODEL_TARGET_SIGNING_OPTION, + VuforiaBackend, +) from tests.mock_vws.utils import ModelTargetEndpoint from tests.mock_vws.utils.assertions import ( assert_model_target_status, @@ -1595,8 +1598,34 @@ def test_unknown_dataset( assert error["target"].startswith("userId:") +# Creating an advanced dataset with a state-based configuration is a +# "signed" request: the real Vuforia signs the trained dataset, and each +# signing consumes the account's Model Target training allowance. The +# allowance is tiny (roughly 20 signings, under ten CI runs' worth), it +# is shared by every CI job and every concurrent run, and it cannot be +# raised or reset by us. Verifying this behavior on every run therefore +# burns the whole allowance within hours and then turns every CI run red +# with ``TRAINING_ALLOWANCE_EXCEEDED`` - which is exactly what happened +# when it ran unconditionally. The equivalent unsigned requests (a +# standard dataset, or an advanced dataset without a state-based +# configuration) consume nothing and stay verified on every run. +_SIGNED_REQUEST_SKIP_REASON = ( + "Signed Model Target requests consume the real Vuforia account's " + "small, shared, non-resettable training allowance, so they are not " + "verified against the real Vuforia by default. Pass " + f"{VERIFY_MODEL_TARGET_SIGNING_OPTION} to verify them, for example " + "after the allowance has recovered. The mock backends always run " + "this test." +) + + class TestStateBasedDatasets: - """Verified fake tests for State-Based Model Targets.""" + """Verified fake tests for State-Based Model Targets. + + The advanced (signed) cases are verified against the real Vuforia + only when ``--verify-model-target-signing`` is given: see + ``_SIGNED_REQUEST_SKIP_REASON``. + """ @staticmethod @pytest.mark.parametrize( @@ -1621,6 +1650,7 @@ class TestStateBasedDatasets: ) def test_state_based_dataset( *, + request: pytest.FixtureRequest, verify_model_target_mock_vuforia: VuforiaBackend, dataset_path: str, view_updates: dict[str, object], @@ -1628,6 +1658,14 @@ def test_state_based_dataset( """State-Based Model Target fields survive a dataset round trip. """ + if ( + verify_model_target_mock_vuforia is VuforiaBackend.REAL + and dataset_path == "/modeltargets/advancedDatasets" + and not request.config.getoption( + name=VERIFY_MODEL_TARGET_SIGNING_OPTION, + ) + ): + pytest.skip(reason=_SIGNED_REQUEST_SKIP_REASON) body = { **_UNAUTHENTICATED_DATASET_REQUEST, "models": [ diff --git a/tests/mock_vws/utils/__init__.py b/tests/mock_vws/utils/__init__.py index 3f405ca85..368c26f8c 100644 --- a/tests/mock_vws/utils/__init__.py +++ b/tests/mock_vws/utils/__init__.py @@ -15,6 +15,8 @@ from mock_vws._constants import ResultCodes +_REQUEST_TIMEOUT_SECONDS = 30 + @beartype def _send_request( @@ -34,7 +36,10 @@ def _send_request( prepared_request = request.prepare() prepared_request.headers = CaseInsensitiveDict(data=headers) session = requests.Session() - requests_response = session.send(request=prepared_request) + requests_response = session.send( + request=prepared_request, + timeout=_REQUEST_TIMEOUT_SECONDS, + ) return Response( text=requests_response.text, url=requests_response.url, diff --git a/uv.lock b/uv.lock index 15fa38766..fe0b2f81c 100644 --- a/uv.lock +++ b/uv.lock @@ -1888,16 +1888,16 @@ wheels = [ [[package]] name = "responses" -version = "0.26.2" +version = "0.26.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyyaml" }, { name = "requests" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f0/1a/4af3e6d659394b809838490b144e4ab8d7ed3b9fecc7ca78f5d2f79b1a3d/responses-0.26.2.tar.gz", hash = "sha256:9c9259b46a8349197edebf43cfa68a87e1a2802ef503ff8b2fecbabc0b45afd8", size = 84030, upload-time = "2026-07-03T16:44:50.325Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/47/f216a33221db8eff328987661cf18371afee89c62a62b434b963d6b509c9/responses-0.26.3.tar.gz", hash = "sha256:b0c11ca8131b8b227b8d5108e6ed39772222bd5aab030ed430e8f99057c4c409", size = 86335, upload-time = "2026-08-26T19:17:24.373Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/28/693e1d9ebf72baa062ded80d837a035b86ce75eda5a269379e9e2b1008a8/responses-0.26.2-py3-none-any.whl", hash = "sha256:6fdfeabd58e5ec473b98dfe02e6d46d3173bd8dd573eff2ccccf1a05a5135364", size = 35609, upload-time = "2026-07-03T16:44:49.1Z" }, + { url = "https://files.pythonhosted.org/packages/6d/86/ca7958de70cb0752350575e98229368a3a2f746a2942034b3364e17312bb/responses-0.26.3-py3-none-any.whl", hash = "sha256:74474f799334ac4f37d93b6437ecc3bb1bb5c77a8d31780a338643be2dce0af8", size = 36289, upload-time = "2026-08-26T19:17:23.176Z" }, ] [[package]]