Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions docs/source/contributing.rst
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,24 @@ Use the following custom ``pytest`` options to skip some tests:
--skip-docker_build_tests
Skip tests for building Docker images

Verifying signed Model Target requests
--------------------------------------

Creating an advanced Model Target dataset with a state-based configuration is a "signed" request: the real Vuforia signs the trained dataset, and each signing consumes the account's Model Target training allowance.
The allowance is small (roughly 20 signings), it is shared by every CI job and every concurrent run, and it cannot be raised or reset.
Verifying signed requests on every run exhausted the allowance within hours and then made every CI run fail with ``TRAINING_ALLOWANCE_EXCEEDED``.

The signed test cases therefore run against the mock backends on every run, but are skipped against the real Vuforia by default.
To verify them against the real Vuforia, for example after the allowance has recovered, opt in with:

.. code-block:: text

--verify-model-target-signing
Run signed Model Target dataset tests against
the real Vuforia

The equivalent unsigned requests (a standard dataset, or an advanced dataset without a state-based configuration) consume no allowance and are verified against the real Vuforia on every run.

Documentation
-------------

Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ optional-dependencies.dev = [
"pydocstyle==6.3",
"pylint[spelling]==4.0.7",
"pylint-per-file-ignores==3.2.1",
"pyproject-fmt==2.28.1",
"pyproject-fmt==2.28.2",
"pyrefly==1.2.0",
"pyright==1.1.411",
"pyroma==5.0.1",
Expand Down
20 changes: 20 additions & 0 deletions tests/mock_vws/fixtures/vuforia_backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,14 @@ class VuforiaBackend(Enum):
}


# Signed Model Target requests (advanced datasets with a state-based
# configuration) consume the Vuforia account's Model Target training
# allowance. The allowance is small, shared across all CI jobs, and
# cannot be raised or reset, so signed requests run against the real
# Vuforia only when this option is given.
VERIFY_MODEL_TARGET_SIGNING_OPTION = "--verify-model-target-signing"


@beartype
def pytest_addoption(parser: pytest.Parser) -> None:
"""
Expand All @@ -387,6 +395,18 @@ def pytest_addoption(parser: pytest.Parser) -> None:
help="Skip tests for building Docker images",
)

parser.addoption(
VERIFY_MODEL_TARGET_SIGNING_OPTION,
action="store_true",
default=False,
help=(
"Run signed Model Target dataset tests against the real "
"Vuforia. These consume the account's small, shared, "
"non-resettable Model Target training allowance, so they "
"run against the mock backends only by default."
),
)


@beartype
def pytest_collection_modifyitems(
Expand Down
42 changes: 40 additions & 2 deletions tests/mock_vws/test_model_target_web_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,10 @@
credentials_for_backend,
get_access_token,
)
from tests.mock_vws.fixtures.vuforia_backends import VuforiaBackend
from tests.mock_vws.fixtures.vuforia_backends import (
VERIFY_MODEL_TARGET_SIGNING_OPTION,
VuforiaBackend,
)
from tests.mock_vws.utils import ModelTargetEndpoint
from tests.mock_vws.utils.assertions import (
assert_model_target_status,
Expand Down Expand Up @@ -1595,8 +1598,34 @@ def test_unknown_dataset(
assert error["target"].startswith("userId:")


# Creating an advanced dataset with a state-based configuration is a
# "signed" request: the real Vuforia signs the trained dataset, and each
# signing consumes the account's Model Target training allowance. The
# allowance is tiny (roughly 20 signings, under ten CI runs' worth), it
# is shared by every CI job and every concurrent run, and it cannot be
# raised or reset by us. Verifying this behavior on every run therefore
# burns the whole allowance within hours and then turns every CI run red
# with ``TRAINING_ALLOWANCE_EXCEEDED`` - which is exactly what happened
# when it ran unconditionally. The equivalent unsigned requests (a
# standard dataset, or an advanced dataset without a state-based
# configuration) consume nothing and stay verified on every run.
_SIGNED_REQUEST_SKIP_REASON = (
"Signed Model Target requests consume the real Vuforia account's "
"small, shared, non-resettable training allowance, so they are not "
"verified against the real Vuforia by default. Pass "
f"{VERIFY_MODEL_TARGET_SIGNING_OPTION} to verify them, for example "
"after the allowance has recovered. The mock backends always run "
"this test."
)


class TestStateBasedDatasets:
"""Verified fake tests for State-Based Model Targets."""
"""Verified fake tests for State-Based Model Targets.

The advanced (signed) cases are verified against the real Vuforia
only when ``--verify-model-target-signing`` is given: see
``_SIGNED_REQUEST_SKIP_REASON``.
"""

@staticmethod
@pytest.mark.parametrize(
Expand All @@ -1621,13 +1650,22 @@ class TestStateBasedDatasets:
)
def test_state_based_dataset(
*,
request: pytest.FixtureRequest,
verify_model_target_mock_vuforia: VuforiaBackend,
dataset_path: str,
view_updates: dict[str, object],
) -> None:
"""State-Based Model Target fields survive a dataset round
trip.
"""
if (
verify_model_target_mock_vuforia is VuforiaBackend.REAL
and dataset_path == "/modeltargets/advancedDatasets"
and not request.config.getoption(
name=VERIFY_MODEL_TARGET_SIGNING_OPTION,
)
):
pytest.skip(reason=_SIGNED_REQUEST_SKIP_REASON)
body = {
**_UNAUTHENTICATED_DATASET_REQUEST,
"models": [
Expand Down
7 changes: 6 additions & 1 deletion tests/mock_vws/utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@

from mock_vws._constants import ResultCodes

_REQUEST_TIMEOUT_SECONDS = 30


@beartype
def _send_request(
Expand All @@ -34,7 +36,10 @@ def _send_request(
prepared_request = request.prepare()
prepared_request.headers = CaseInsensitiveDict(data=headers)
session = requests.Session()
requests_response = session.send(request=prepared_request)
requests_response = session.send(
request=prepared_request,
timeout=_REQUEST_TIMEOUT_SECONDS,
)
return Response(
text=requests_response.text,
url=requests_response.url,
Expand Down
Loading