diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 09faeb7b3..24ab6e067 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -124,6 +124,24 @@ Use the following custom ``pytest`` options to skip some tests: --skip-docker_build_tests Skip tests for building Docker images +Verifying signed Model Target requests +-------------------------------------- + +Creating an advanced Model Target dataset with a state-based configuration is a "signed" request: the real Vuforia signs the trained dataset, and each signing consumes the account's Model Target training allowance. +The allowance is small (roughly 20 signings), it is shared by every CI job and every concurrent run, and it cannot be raised or reset. +Verifying signed requests on every run exhausted the allowance within hours and then made every CI run fail with ``TRAINING_ALLOWANCE_EXCEEDED``. + +The signed test cases therefore run against the mock backends on every run, but are skipped against the real Vuforia by default. +To verify them against the real Vuforia, for example after the allowance has recovered, opt in with: + +.. code-block:: text + + --verify-model-target-signing + Run signed Model Target dataset tests against + the real Vuforia + +The equivalent unsigned requests (a standard dataset, or an advanced dataset without a state-based configuration) consume no allowance and are verified against the real Vuforia on every run. + Documentation ------------- diff --git a/pyproject.toml b/pyproject.toml index 51f38d769..821c5811e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,7 +108,7 @@ optional-dependencies.dev = [ "tenacity==9.1.4", "towncrier==25.8.0", "ty==0.0.74", - "types-docker==7.2.0.20260819", + "types-docker==7.2.0.20260827", "types-pyyaml==6.0.12.20260815", "types-requests==2.33.0.20260712", "urllib3==2.7.0", diff --git a/tests/mock_vws/fixtures/vuforia_backends.py b/tests/mock_vws/fixtures/vuforia_backends.py index 11dae25f5..533e9dea0 100644 --- a/tests/mock_vws/fixtures/vuforia_backends.py +++ b/tests/mock_vws/fixtures/vuforia_backends.py @@ -371,6 +371,14 @@ class VuforiaBackend(Enum): } +# Signed Model Target requests (advanced datasets with a state-based +# configuration) consume the Vuforia account's Model Target training +# allowance. The allowance is small, shared across all CI jobs, and +# cannot be raised or reset, so signed requests run against the real +# Vuforia only when this option is given. +VERIFY_MODEL_TARGET_SIGNING_OPTION = "--verify-model-target-signing" + + @beartype def pytest_addoption(parser: pytest.Parser) -> None: """ @@ -387,6 +395,18 @@ def pytest_addoption(parser: pytest.Parser) -> None: help="Skip tests for building Docker images", ) + parser.addoption( + VERIFY_MODEL_TARGET_SIGNING_OPTION, + action="store_true", + default=False, + help=( + "Run signed Model Target dataset tests against the real " + "Vuforia. These consume the account's small, shared, " + "non-resettable Model Target training allowance, so they " + "run against the mock backends only by default." + ), + ) + @beartype def pytest_collection_modifyitems( diff --git a/tests/mock_vws/test_model_target_web_api.py b/tests/mock_vws/test_model_target_web_api.py index 138340abf..1e930040e 100644 --- a/tests/mock_vws/test_model_target_web_api.py +++ b/tests/mock_vws/test_model_target_web_api.py @@ -31,7 +31,10 @@ credentials_for_backend, get_access_token, ) -from tests.mock_vws.fixtures.vuforia_backends import VuforiaBackend +from tests.mock_vws.fixtures.vuforia_backends import ( + VERIFY_MODEL_TARGET_SIGNING_OPTION, + VuforiaBackend, +) from tests.mock_vws.utils import ModelTargetEndpoint from tests.mock_vws.utils.assertions import ( assert_model_target_status, @@ -1595,8 +1598,34 @@ def test_unknown_dataset( assert error["target"].startswith("userId:") +# Creating an advanced dataset with a state-based configuration is a +# "signed" request: the real Vuforia signs the trained dataset, and each +# signing consumes the account's Model Target training allowance. The +# allowance is tiny (roughly 20 signings, under ten CI runs' worth), it +# is shared by every CI job and every concurrent run, and it cannot be +# raised or reset by us. Verifying this behavior on every run therefore +# burns the whole allowance within hours and then turns every CI run red +# with ``TRAINING_ALLOWANCE_EXCEEDED`` - which is exactly what happened +# when it ran unconditionally. The equivalent unsigned requests (a +# standard dataset, or an advanced dataset without a state-based +# configuration) consume nothing and stay verified on every run. +_SIGNED_REQUEST_SKIP_REASON = ( + "Signed Model Target requests consume the real Vuforia account's " + "small, shared, non-resettable training allowance, so they are not " + "verified against the real Vuforia by default. Pass " + f"{VERIFY_MODEL_TARGET_SIGNING_OPTION} to verify them, for example " + "after the allowance has recovered. The mock backends always run " + "this test." +) + + class TestStateBasedDatasets: - """Verified fake tests for State-Based Model Targets.""" + """Verified fake tests for State-Based Model Targets. + + The advanced (signed) cases are verified against the real Vuforia + only when ``--verify-model-target-signing`` is given: see + ``_SIGNED_REQUEST_SKIP_REASON``. + """ @staticmethod @pytest.mark.parametrize( @@ -1621,6 +1650,7 @@ class TestStateBasedDatasets: ) def test_state_based_dataset( *, + request: pytest.FixtureRequest, verify_model_target_mock_vuforia: VuforiaBackend, dataset_path: str, view_updates: dict[str, object], @@ -1628,6 +1658,14 @@ def test_state_based_dataset( """State-Based Model Target fields survive a dataset round trip. """ + if ( + verify_model_target_mock_vuforia is VuforiaBackend.REAL + and dataset_path == "/modeltargets/advancedDatasets" + and not request.config.getoption( + name=VERIFY_MODEL_TARGET_SIGNING_OPTION, + ) + ): + pytest.skip(reason=_SIGNED_REQUEST_SKIP_REASON) body = { **_UNAUTHENTICATED_DATASET_REQUEST, "models": [ diff --git a/tests/mock_vws/utils/__init__.py b/tests/mock_vws/utils/__init__.py index 3f405ca85..368c26f8c 100644 --- a/tests/mock_vws/utils/__init__.py +++ b/tests/mock_vws/utils/__init__.py @@ -15,6 +15,8 @@ from mock_vws._constants import ResultCodes +_REQUEST_TIMEOUT_SECONDS = 30 + @beartype def _send_request( @@ -34,7 +36,10 @@ def _send_request( prepared_request = request.prepare() prepared_request.headers = CaseInsensitiveDict(data=headers) session = requests.Session() - requests_response = session.send(request=prepared_request) + requests_response = session.send( + request=prepared_request, + timeout=_REQUEST_TIMEOUT_SECONDS, + ) return Response( text=requests_response.text, url=requests_response.url, diff --git a/uv.lock b/uv.lock index 15fa38766..42a819485 100644 --- a/uv.lock +++ b/uv.lock @@ -2607,15 +2607,15 @@ wheels = [ [[package]] name = "types-docker" -version = "7.2.0.20260819" +version = "7.2.0.20260827" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "types-requests" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f3/b6/4ccc887c1ef5974699d4e0c1904d72534f99c7bb6c88bc24b8000a4c81bd/types_docker-7.2.0.20260819.tar.gz", hash = "sha256:3fe9a80f1f1ba271d9188b58f46c743eede65e1c46bb82ca87101725a2dc7a22", size = 36846, upload-time = "2026-08-19T02:47:25.481Z" } +sdist = { url = "https://files.pythonhosted.org/packages/66/92/ebed321b5e0d5f6b2efc2fee34e99961d1425600d3b1629f0684176bc204/types_docker-7.2.0.20260827.tar.gz", hash = "sha256:6e1674ec12dd4d09d6437f735dcf6b3c61e730c330623aacf92b492e1444094f", size = 36929, upload-time = "2026-08-27T12:07:36.723Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/ea/4c1dd437b82b55c6333e31617cc9befd07d19392404d861e290cc8d6b8c3/types_docker-7.2.0.20260819-py3-none-any.whl", hash = "sha256:b25951651f2bdad88e194067bb8a523d2963a316caef699a2c2a4e30c3394ae2", size = 51258, upload-time = "2026-08-19T02:47:24.4Z" }, + { url = "https://files.pythonhosted.org/packages/0e/62/40a2b11c2be94d7288d7c0c5ca673d2b9d11f2930f58ec9d5d98380092ba/types_docker-7.2.0.20260827-py3-none-any.whl", hash = "sha256:ecc2b403ec086552a298232b92b662fb195d0e749f155c0b707e59fdedb5bfdf", size = 51269, upload-time = "2026-08-27T12:07:35.671Z" }, ] [[package]] @@ -2883,7 +2883,7 @@ requires-dist = [ { name = "towncrier", marker = "extra == 'dev'", specifier = "==25.8.0" }, { name = "towncrier", marker = "extra == 'release'", specifier = "==25.8.0" }, { name = "ty", marker = "extra == 'dev'", specifier = "==0.0.74" }, - { name = "types-docker", marker = "extra == 'dev'", specifier = "==7.2.0.20260819" }, + { name = "types-docker", marker = "extra == 'dev'", specifier = "==7.2.0.20260827" }, { name = "types-pyyaml", marker = "extra == 'dev'", specifier = "==6.0.12.20260815" }, { name = "types-requests", marker = "extra == 'dev'", specifier = "==2.33.0.20260712" }, { name = "tzdata", marker = "sys_platform == 'win32'" },