diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 09faeb7b3..e85ce8384 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -124,6 +124,25 @@ Use the following custom ``pytest`` options to skip some tests: --skip-docker_build_tests Skip tests for building Docker images +Verifying signed Model Target requests +-------------------------------------- + +Creating an advanced Model Target dataset with a state-based configuration is a "signed" request: the real Vuforia signs the trained dataset, and each signing consumes the account's Model Target training allowance. +The allowance is small (roughly 20 signings), it is shared by every CI job and every concurrent run, and it cannot be raised or reset. +Verifying signed requests on every run exhausted the allowance within hours and then made every CI run fail with ``TRAINING_ALLOWANCE_EXCEEDED``. + +The signed test cases therefore run against the mock backends on every run, but are skipped against the real Vuforia by default. +To verify them against the real Vuforia, for example after the allowance has recovered, opt in with: + +.. code-block:: text + + --verify-model-target-signing + Run signed Model Target dataset tests against + the real Vuforia + +The equivalent unsigned requests (a standard dataset, or an advanced dataset without a state-based configuration) are far cheaper and are verified against the real Vuforia on every run. +With enough traffic even unsigned dataset creation can be rejected with ``TRAINING_ALLOWANCE_EXCEEDED``; an unexpected allowance rejection is reported as an expected failure rather than a test failure, and the affected tests pass again automatically once the allowance recovers. + Documentation ------------- diff --git a/tests/mock_vws/fixtures/vuforia_backends.py b/tests/mock_vws/fixtures/vuforia_backends.py index 11dae25f5..533e9dea0 100644 --- a/tests/mock_vws/fixtures/vuforia_backends.py +++ b/tests/mock_vws/fixtures/vuforia_backends.py @@ -371,6 +371,14 @@ class VuforiaBackend(Enum): } +# Signed Model Target requests (advanced datasets with a state-based +# configuration) consume the Vuforia account's Model Target training +# allowance. The allowance is small, shared across all CI jobs, and +# cannot be raised or reset, so signed requests run against the real +# Vuforia only when this option is given. +VERIFY_MODEL_TARGET_SIGNING_OPTION = "--verify-model-target-signing" + + @beartype def pytest_addoption(parser: pytest.Parser) -> None: """ @@ -387,6 +395,18 @@ def pytest_addoption(parser: pytest.Parser) -> None: help="Skip tests for building Docker images", ) + parser.addoption( + VERIFY_MODEL_TARGET_SIGNING_OPTION, + action="store_true", + default=False, + help=( + "Run signed Model Target dataset tests against the real " + "Vuforia. These consume the account's small, shared, " + "non-resettable Model Target training allowance, so they " + "run against the mock backends only by default." + ), + ) + @beartype def pytest_collection_modifyitems( diff --git a/tests/mock_vws/test_model_target_web_api.py b/tests/mock_vws/test_model_target_web_api.py index 138340abf..69bab0f0d 100644 --- a/tests/mock_vws/test_model_target_web_api.py +++ b/tests/mock_vws/test_model_target_web_api.py @@ -31,7 +31,10 @@ credentials_for_backend, get_access_token, ) -from tests.mock_vws.fixtures.vuforia_backends import VuforiaBackend +from tests.mock_vws.fixtures.vuforia_backends import ( + VERIFY_MODEL_TARGET_SIGNING_OPTION, + VuforiaBackend, +) from tests.mock_vws.utils import ModelTargetEndpoint from tests.mock_vws.utils.assertions import ( assert_model_target_status, @@ -1595,8 +1598,37 @@ def test_unknown_dataset( assert error["target"].startswith("userId:") +# Creating an advanced dataset with a state-based configuration is a +# "signed" request: the real Vuforia signs the trained dataset, and each +# signing consumes the account's Model Target training allowance. The +# allowance is tiny (roughly 20 signings, under ten CI runs' worth), it +# is shared by every CI job and every concurrent run, and it cannot be +# raised or reset by us. Verifying this behavior on every run therefore +# burns the whole allowance within hours and then turns every CI run red +# with ``TRAINING_ALLOWANCE_EXCEEDED`` - which is exactly what happened +# when it ran unconditionally. The equivalent unsigned requests (a +# standard dataset, or an advanced dataset without a state-based +# configuration) are far cheaper and stay enabled, though with enough +# traffic they can also hit the allowance; an unexpected allowance +# rejection is reported as an expected failure by +# ``assert_model_target_status`` rather than failing the run. +_SIGNED_REQUEST_SKIP_REASON = ( + "Signed Model Target requests consume the real Vuforia account's " + "small, shared, non-resettable training allowance, so they are not " + "verified against the real Vuforia by default. Pass " + f"{VERIFY_MODEL_TARGET_SIGNING_OPTION} to verify them, for example " + "after the allowance has recovered. The mock backends always run " + "this test." +) + + class TestStateBasedDatasets: - """Verified fake tests for State-Based Model Targets.""" + """Verified fake tests for State-Based Model Targets. + + The advanced (signed) cases are verified against the real Vuforia + only when ``--verify-model-target-signing`` is given: see + ``_SIGNED_REQUEST_SKIP_REASON``. + """ @staticmethod @pytest.mark.parametrize( @@ -1621,6 +1653,7 @@ class TestStateBasedDatasets: ) def test_state_based_dataset( *, + request: pytest.FixtureRequest, verify_model_target_mock_vuforia: VuforiaBackend, dataset_path: str, view_updates: dict[str, object], @@ -1628,6 +1661,14 @@ def test_state_based_dataset( """State-Based Model Target fields survive a dataset round trip. """ + if ( + verify_model_target_mock_vuforia is VuforiaBackend.REAL + and dataset_path == "/modeltargets/advancedDatasets" + and not request.config.getoption( + name=VERIFY_MODEL_TARGET_SIGNING_OPTION, + ) + ): + pytest.skip(reason=_SIGNED_REQUEST_SKIP_REASON) body = { **_UNAUTHENTICATED_DATASET_REQUEST, "models": [ @@ -2673,8 +2714,9 @@ def test_multiple_expected_statuses() -> None: @staticmethod def test_training_allowance_exceeded() -> None: - """An exhausted account allowance is called out as such, in the - first line so that a truncated CI summary still shows it. + """An exhausted account allowance is an expected failure, called + out as such in the first line so that a truncated CI summary + still shows it. """ response = _fake_response( status_code=HTTPStatus.UNPROCESSABLE_ENTITY, @@ -2683,7 +2725,7 @@ def test_training_allowance_exceeded() -> None: '"message":"Signing quota reached","target":"7635391"}}' ), ) - with pytest.raises(expected_exception=AssertionError) as exc: + with pytest.raises(expected_exception=pytest.xfail.Exception) as exc: assert_model_target_status( response=response, status_codes=HTTPStatus.CREATED, @@ -2697,3 +2739,4 @@ def test_training_allowance_exceeded() -> None: ) assert "MODEL_TARGET_VUFORIA_CLIENT_ID" in message assert "has to be raised, or reset, on the Vuforia account" in message + assert "expected failure" in message diff --git a/tests/mock_vws/test_requests_mock_usage.py b/tests/mock_vws/test_requests_mock_usage.py index 71225acc2..d71c0944a 100644 --- a/tests/mock_vws/test_requests_mock_usage.py +++ b/tests/mock_vws/test_requests_mock_usage.py @@ -18,7 +18,7 @@ from beartype import beartype from freezegun import freeze_time from PIL import Image -from vws import VWS, CloudRecoService +from vws import VWS, CloudRecoService, VuMarkService from vws.exceptions.vws_exceptions import ( ProjectSuspendedError, RequestQuotaReachedError, @@ -27,6 +27,7 @@ ) from vws.reports import TargetStatuses from vws.transports import HTTPXTransport +from vws.vumark_accept import VuMarkAccept from vws_auth_tools import authorization_header, rfc_1123_date from mock_vws import MissingSchemeError, MockVWS @@ -264,6 +265,28 @@ def test_delay_allows_completion() -> None: ) assert response.status_code is not None + @staticmethod + def test_delay_without_timeout() -> None: + """A request without a timeout waits for the configured delay.""" + calls: list[float] = [] + with MockVWS( + response_delay_seconds=0.1, + sleep_fn=calls.append, + ): + # Omitting the timeout is the behavior under test. + # pylint: disable-next=missing-timeout + response = requests.get( # noqa: S113 + url="https://vws.vuforia.com/summary", + headers={ + "Date": rfc_1123_date(), + "Authorization": "bad_auth_token", + }, + data=b"", + ) + + assert response.status_code is not None + assert calls == [0.1] + @staticmethod def test_delay_with_tuple_timeout() -> None: """ @@ -1517,6 +1540,33 @@ def test_duplicate_vumark_keys() -> None: ): mock.add_vumark_database(vumark_database=bad_database) + @staticmethod + def test_vumark_database_added_before_entering() -> None: + """A VuMark database added before the mock starts is available + while the mock is running. + """ + database = VuMarkDatabase(database_name="vumark-before-enter") + mock = MockVWS() + mock.add_vumark_database(vumark_database=database) + conflicting_database = VuMarkDatabase( + database_name="vumark-before-enter", + ) + expected_message = ( + "All names must be unique. " + "There is already a database with the name " + '"vumark-before-enter".' + ) + with ( + mock, + pytest.raises( + expected_exception=ValueError, + match=expected_message + "$", + ), + ): + mock.add_vumark_database( + vumark_database=conflicting_database, + ) + class TestContextManagerReuse: """Tests for reusing a ``MockVWS`` instance as a context manager.""" @@ -2252,6 +2302,29 @@ def get_database_name() -> str: assert get_database_name() == database.database_name + @staticmethod + def test_vumark_database_added_before_decorating() -> None: + """VuMark databases are available within a decorated function.""" + vumark_target = VuMarkTarget(name="test-target") + database = VuMarkDatabase(vumark_targets={vumark_target}) + mock = MockVWS() + mock.add_vumark_database(vumark_database=database) + + @mock + def generate_vumark_instance() -> bytes: + """Generate a VuMark instance from the configured database.""" + client = VuMarkService( + server_access_key=database.server_access_key, + server_secret_key=database.server_secret_key, + ) + return client.generate_vumark_instance( + target_id=vumark_target.target_id, + instance_id=uuid.uuid4().hex, + accept=VuMarkAccept.PNG, + ) + + assert generate_vumark_instance().startswith(b"\x89PNG") + @staticmethod def test_options_are_used(image_file_failed_state: io.BytesIO) -> None: """Options given to the mock are used within the decorated @@ -2281,6 +2354,31 @@ def add_target() -> TargetStatuses: # processed immediately. assert add_target() == TargetStatuses.FAILED + @staticmethod + def test_vumark_targets_are_restored() -> None: + """VuMark targets changed during a call of a decorated function + are put back as they were afterwards, just as Cloud targets + are. + """ + vumark_target = VuMarkTarget(name="existing-target") + database = VuMarkDatabase(vumark_targets={vumark_target}) + mock = MockVWS() + mock.add_vumark_database(vumark_database=database) + + temporary_target = VuMarkTarget(name="temporary") + + @mock + def add_temporary_target() -> None: + """Add a target to the database object directly.""" + database.vumark_targets.add(temporary_target) + assert database.vumark_targets == { + vumark_target, + temporary_target, + } + + add_temporary_target() + assert database.vumark_targets == {vumark_target} + @staticmethod def test_each_call_is_isolated(high_quality_image: io.BytesIO) -> None: """Each call of a decorated function has its own targets. diff --git a/tests/mock_vws/utils/__init__.py b/tests/mock_vws/utils/__init__.py index 3f405ca85..368c26f8c 100644 --- a/tests/mock_vws/utils/__init__.py +++ b/tests/mock_vws/utils/__init__.py @@ -15,6 +15,8 @@ from mock_vws._constants import ResultCodes +_REQUEST_TIMEOUT_SECONDS = 30 + @beartype def _send_request( @@ -34,7 +36,10 @@ def _send_request( prepared_request = request.prepare() prepared_request.headers = CaseInsensitiveDict(data=headers) session = requests.Session() - requests_response = session.send(request=prepared_request) + requests_response = session.send( + request=prepared_request, + timeout=_REQUEST_TIMEOUT_SECONDS, + ) return Response( text=requests_response.text, url=requests_response.url, diff --git a/tests/mock_vws/utils/assertions.py b/tests/mock_vws/utils/assertions.py index ec550fb08..ced303885 100644 --- a/tests/mock_vws/utils/assertions.py +++ b/tests/mock_vws/utils/assertions.py @@ -10,6 +10,7 @@ from string import hexdigits from zoneinfo import ZoneInfo +import pytest import requests from beartype import beartype from vws.response import Response @@ -185,9 +186,14 @@ def assert_vws_response( so the allowance is consumed across all jobs and all concurrent runs. The allowance has to be raised, or reset, on the Vuforia account for this - test to pass against the real backend again. Signed dataset types, such as - an advanced dataset with a state-based configuration, exhaust it first: the - equivalent unsigned request keeps succeeding while this one fails.""", + request to succeed against the real backend again. Signed dataset types, + such as an advanced dataset with a state-based configuration, exhaust it + first, but with enough traffic unsigned dataset creation is rejected too. + + An unexpected allowance rejection is reported as an expected failure + (xfail) rather than a test failure, so that the shared account running + dry does not turn CI red: the test passes again automatically once the + allowance recovers.""", ) @@ -214,6 +220,11 @@ def assert_model_target_status( Raises: AssertionError: The response does not have one of the expected status codes. + Exception: The response unexpectedly reports a + ``TRAINING_ALLOWANCE_EXCEEDED`` rejection. That is the shared + Vuforia account running out of Model Target training allowance, + not a fault in the code under test, so it is an expected failure + rather than a test failure. """ expected = ( frozenset({status_codes}) @@ -246,6 +257,8 @@ def assert_model_target_status( *explanation, ], ) + if allowance_exceeded: + pytest.xfail(reason=message) raise AssertionError(message)