diff --git a/sagemaker-mlops/tests/integ/conftest.py b/sagemaker-mlops/tests/integ/conftest.py index a0981c9b5f..537e18c879 100644 --- a/sagemaker-mlops/tests/integ/conftest.py +++ b/sagemaker-mlops/tests/integ/conftest.py @@ -69,6 +69,11 @@ # ``ClientError: (Throttling) ... Rate exceeded``. This mitigation is a purely # test-harness concurrency fix and is intentionally identical to the block in # the sagemaker-train / sagemaker-serve integ conftests. +# +# Throttling that still exhausts the adaptive retry budget is deliberately left +# to fail the test loudly (rather than being converted to a skip), so a +# persistent rate-limit regression stays visible instead of silently +# disappearing from the results. # --------------------------------------------------------------------------- # botocore adaptive retry settings for throttling-prone IAM validation calls. @@ -77,11 +82,6 @@ _RETRY_MODE = "adaptive" _MAX_ATTEMPTS = "10" -# Only tolerate throttling on the IAM permission simulation used during role -# validation, so unrelated throttling still fails loudly. -_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded") -_SIMULATE_OP = "SimulatePrincipalPolicy" - @pytest.fixture(autouse=True, scope="session") def _configure_boto_adaptive_retries(): @@ -102,40 +102,6 @@ def _configure_boto_adaptive_retries(): os.environ[key] = value -def _is_simulate_policy_throttle(exc): - """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError.""" - from botocore.exceptions import ClientError - - if not isinstance(exc, ClientError): - return False - error_code = exc.response.get("Error", {}).get("Code", "") - operation = getattr(exc, "operation_name", "") or "" - return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_makereport(item, call): - """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that - survives the adaptive retries under heavy concurrency. Applies to setup and - call phases, since role resolution frequently happens in fixture setup.""" - outcome = yield - report = outcome.get_result() - - if report.when not in ("setup", "call") or not report.failed: - return - - exc = call.excinfo.value if call.excinfo else None - # Walk the exception chain so a wrapped ClientError is still detected. - while exc is not None: - if _is_simulate_policy_throttle(exc): - report.outcome = "skipped" - report.wasxfail = ( - "iam:SimulatePrincipalPolicy throttled under concurrent test load" - ) - break - exc = exc.__cause__ or exc.__context__ - - # --------------------------------------------------------------------------- # CLI options # --------------------------------------------------------------------------- diff --git a/sagemaker-serve/tests/integ/conftest.py b/sagemaker-serve/tests/integ/conftest.py index 6ffc1a65f1..1609fe375a 100644 --- a/sagemaker-serve/tests/integ/conftest.py +++ b/sagemaker-serve/tests/integ/conftest.py @@ -30,10 +30,10 @@ default 4-attempt retry policy). ``adaptive`` mode also adds client-side rate limiting to smooth bursts. -* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual - ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive - retries, convert the failure into a skip so a transient rate limit never reds - the build. Genuine failures are untouched. +Throttling that still exhausts the adaptive retry budget is deliberately left to +fail the test loudly (rather than being converted to a skip), so a persistent +rate-limit regression stays visible instead of silently disappearing from the +results. """ from __future__ import absolute_import @@ -47,11 +47,6 @@ _RETRY_MODE = "adaptive" _MAX_ATTEMPTS = "10" -# Only tolerate throttling on the IAM permission simulation used during role -# validation, so unrelated throttling still fails loudly. -_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded") -_SIMULATE_OP = "SimulatePrincipalPolicy" - @pytest.fixture(autouse=True, scope="session") def _configure_boto_adaptive_retries(): @@ -70,37 +65,3 @@ def _configure_boto_adaptive_retries(): os.environ.pop(key, None) else: os.environ[key] = value - - -def _is_simulate_policy_throttle(exc): - """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError.""" - from botocore.exceptions import ClientError - - if not isinstance(exc, ClientError): - return False - error_code = exc.response.get("Error", {}).get("Code", "") - operation = getattr(exc, "operation_name", "") or "" - return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_makereport(item, call): - """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that - survives the adaptive retries under heavy concurrency. Applies to setup and - call phases, since role resolution frequently happens in fixture setup.""" - outcome = yield - report = outcome.get_result() - - if report.when not in ("setup", "call") or not report.failed: - return - - exc = call.excinfo.value if call.excinfo else None - # Walk the exception chain so a wrapped ClientError is still detected. - while exc is not None: - if _is_simulate_policy_throttle(exc): - report.outcome = "skipped" - report.wasxfail = ( - "iam:SimulatePrincipalPolicy throttled under concurrent test load" - ) - break - exc = exc.__cause__ or exc.__context__ diff --git a/sagemaker-train/tests/integ/conftest.py b/sagemaker-train/tests/integ/conftest.py index 87c861092a..de5e45aef8 100644 --- a/sagemaker-train/tests/integ/conftest.py +++ b/sagemaker-train/tests/integ/conftest.py @@ -31,11 +31,10 @@ its own). ``adaptive`` mode adds client-side rate limiting so bursts of ``SimulatePrincipalPolicy`` calls ride out transient throttling. -* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual - ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive - retries, convert the failure into a skip so a transient rate limit never reds - the build. Genuine failures (and throttling on any other operation) are - untouched. +Throttling that still exhausts the adaptive retry budget is deliberately left to +fail the test loudly (rather than being converted to a skip), so a persistent +rate-limit regression stays visible instead of silently disappearing from the +results. """ from __future__ import absolute_import @@ -49,11 +48,6 @@ _RETRY_MODE = "adaptive" _MAX_ATTEMPTS = "10" -# Only tolerate throttling on the IAM permission simulation used during role -# validation, so unrelated throttling still fails loudly. -_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded") -_SIMULATE_OP = "SimulatePrincipalPolicy" - @pytest.fixture(autouse=True, scope="session") def _configure_boto_adaptive_retries(): @@ -72,37 +66,3 @@ def _configure_boto_adaptive_retries(): os.environ.pop(key, None) else: os.environ[key] = value - - -def _is_simulate_policy_throttle(exc): - """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError.""" - from botocore.exceptions import ClientError - - if not isinstance(exc, ClientError): - return False - error_code = exc.response.get("Error", {}).get("Code", "") - operation = getattr(exc, "operation_name", "") or "" - return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_makereport(item, call): - """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that - survives the adaptive retries under heavy concurrency. Applies to setup and - call phases, since role resolution frequently happens in fixture setup.""" - outcome = yield - report = outcome.get_result() - - if report.when not in ("setup", "call") or not report.failed: - return - - exc = call.excinfo.value if call.excinfo else None - # Walk the exception chain so a wrapped ClientError is still detected. - while exc is not None: - if _is_simulate_policy_throttle(exc): - report.outcome = "skipped" - report.wasxfail = ( - "iam:SimulatePrincipalPolicy throttled under concurrent test load" - ) - break - exc = exc.__cause__ or exc.__context__