test(integ): let exhausted IAM throttling fail instead of skipping

sagemaker-mlops/tests/integ/conftest.py

@@ -69,6 +69,11 @@
 # ``ClientError: (Throttling) ... Rate exceeded``. This mitigation is a purely
 # test-harness concurrency fix and is intentionally identical to the block in
 # the sagemaker-train / sagemaker-serve integ conftests.
+#
+# Throttling that still exhausts the adaptive retry budget is deliberately left
+# to fail the test loudly (rather than being converted to a skip), so a
+# persistent rate-limit regression stays visible instead of silently
+# disappearing from the results.
 # ---------------------------------------------------------------------------
 
 # botocore adaptive retry settings for throttling-prone IAM validation calls.
@@ -77,11 +82,6 @@
 _RETRY_MODE = "adaptive"
 _MAX_ATTEMPTS = "10"
 
-# Only tolerate throttling on the IAM permission simulation used during role
-# validation, so unrelated throttling still fails loudly.
-_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
-_SIMULATE_OP = "SimulatePrincipalPolicy"
-
 
 @pytest.fixture(autouse=True, scope="session")
 def _configure_boto_adaptive_retries():
@@ -102,40 +102,6 @@ def _configure_boto_adaptive_retries():
             os.environ[key] = value
 
 
-def _is_simulate_policy_throttle(exc):
-    """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
-    from botocore.exceptions import ClientError
-
-    if not isinstance(exc, ClientError):
-        return False
-    error_code = exc.response.get("Error", {}).get("Code", "")
-    operation = getattr(exc, "operation_name", "") or ""
-    return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP
-
-
-@pytest.hookimpl(hookwrapper=True)
-def pytest_runtest_makereport(item, call):
-    """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
-    survives the adaptive retries under heavy concurrency. Applies to setup and
-    call phases, since role resolution frequently happens in fixture setup."""
-    outcome = yield
-    report = outcome.get_result()
-
-    if report.when not in ("setup", "call") or not report.failed:
-        return
-
-    exc = call.excinfo.value if call.excinfo else None
-    # Walk the exception chain so a wrapped ClientError is still detected.
-    while exc is not None:
-        if _is_simulate_policy_throttle(exc):
-            report.outcome = "skipped"
-            report.wasxfail = (
-                "iam:SimulatePrincipalPolicy throttled under concurrent test load"
-            )
-            break
-        exc = exc.__cause__ or exc.__context__
-
-
 # ---------------------------------------------------------------------------
 # CLI options
 # ---------------------------------------------------------------------------

sagemaker-serve/tests/integ/conftest.py

@@ -30,10 +30,10 @@
   default 4-attempt retry policy). ``adaptive`` mode also adds client-side rate
   limiting to smooth bursts.
 
-* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
-  ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
-  retries, convert the failure into a skip so a transient rate limit never reds
-  the build. Genuine failures are untouched.
+Throttling that still exhausts the adaptive retry budget is deliberately left to
+fail the test loudly (rather than being converted to a skip), so a persistent
+rate-limit regression stays visible instead of silently disappearing from the
+results.
 """
 from __future__ import absolute_import
 
@@ -47,11 +47,6 @@
 _RETRY_MODE = "adaptive"
 _MAX_ATTEMPTS = "10"
 
-# Only tolerate throttling on the IAM permission simulation used during role
-# validation, so unrelated throttling still fails loudly.
-_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
-_SIMULATE_OP = "SimulatePrincipalPolicy"
-
 
 @pytest.fixture(autouse=True, scope="session")
 def _configure_boto_adaptive_retries():
@@ -70,37 +65,3 @@ def _configure_boto_adaptive_retries():
             os.environ.pop(key, None)
         else:
             os.environ[key] = value
-
-
-def _is_simulate_policy_throttle(exc):
-    """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
-    from botocore.exceptions import ClientError
-
-    if not isinstance(exc, ClientError):
-        return False
-    error_code = exc.response.get("Error", {}).get("Code", "")
-    operation = getattr(exc, "operation_name", "") or ""
-    return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP
-
-
-@pytest.hookimpl(hookwrapper=True)
-def pytest_runtest_makereport(item, call):
-    """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
-    survives the adaptive retries under heavy concurrency. Applies to setup and
-    call phases, since role resolution frequently happens in fixture setup."""
-    outcome = yield
-    report = outcome.get_result()
-
-    if report.when not in ("setup", "call") or not report.failed:
-        return
-
-    exc = call.excinfo.value if call.excinfo else None
-    # Walk the exception chain so a wrapped ClientError is still detected.
-    while exc is not None:
-        if _is_simulate_policy_throttle(exc):
-            report.outcome = "skipped"
-            report.wasxfail = (
-                "iam:SimulatePrincipalPolicy throttled under concurrent test load"
-            )
-            break
-        exc = exc.__cause__ or exc.__context__

sagemaker-train/tests/integ/conftest.py

@@ -31,11 +31,10 @@
   its own). ``adaptive`` mode adds client-side rate limiting so bursts of
   ``SimulatePrincipalPolicy`` calls ride out transient throttling.
 
-* ``pytest_runtest_makereport`` — a belt-and-suspenders fallback. If a residual
-  ``SimulatePrincipalPolicy`` throttling error still escapes after the adaptive
-  retries, convert the failure into a skip so a transient rate limit never reds
-  the build. Genuine failures (and throttling on any other operation) are
-  untouched.
+Throttling that still exhausts the adaptive retry budget is deliberately left to
+fail the test loudly (rather than being converted to a skip), so a persistent
+rate-limit regression stays visible instead of silently disappearing from the
+results.
 """
 from __future__ import absolute_import
 
@@ -49,11 +48,6 @@
 _RETRY_MODE = "adaptive"
 _MAX_ATTEMPTS = "10"
 
-# Only tolerate throttling on the IAM permission simulation used during role
-# validation, so unrelated throttling still fails loudly.
-_THROTTLE_ERROR_CODES = ("Throttling", "ThrottlingException", "RequestLimitExceeded")
-_SIMULATE_OP = "SimulatePrincipalPolicy"
-
 
 @pytest.fixture(autouse=True, scope="session")
 def _configure_boto_adaptive_retries():
@@ -72,37 +66,3 @@ def _configure_boto_adaptive_retries():
             os.environ.pop(key, None)
         else:
             os.environ[key] = value
-
-
-def _is_simulate_policy_throttle(exc):
-    """Return True if ``exc`` is a SimulatePrincipalPolicy throttling ClientError."""
-    from botocore.exceptions import ClientError
-
-    if not isinstance(exc, ClientError):
-        return False
-    error_code = exc.response.get("Error", {}).get("Code", "")
-    operation = getattr(exc, "operation_name", "") or ""
-    return error_code in _THROTTLE_ERROR_CODES and operation == _SIMULATE_OP
-
-
-@pytest.hookimpl(hookwrapper=True)
-def pytest_runtest_makereport(item, call):
-    """Skip (rather than fail) on residual SimulatePrincipalPolicy throttling that
-    survives the adaptive retries under heavy concurrency. Applies to setup and
-    call phases, since role resolution frequently happens in fixture setup."""
-    outcome = yield
-    report = outcome.get_result()
-
-    if report.when not in ("setup", "call") or not report.failed:
-        return
-
-    exc = call.excinfo.value if call.excinfo else None
-    # Walk the exception chain so a wrapped ClientError is still detected.
-    while exc is not None:
-        if _is_simulate_policy_throttle(exc):
-            report.outcome = "skipped"
-            report.wasxfail = (
-                "iam:SimulatePrincipalPolicy throttled under concurrent test load"
-            )
-            break
-        exc = exc.__cause__ or exc.__context__