From c6f1668ec6d337a58872af0ddb32572f71cb15c0 Mon Sep 17 00:00:00 2001 From: Alexey Shalaev <75322386+AlexeyShalaev@users.noreply.github.com> Date: Mon, 7 Sep 2026 00:25:55 +0300 Subject: [PATCH] fix: give the attempt ceiling its own retryable kind and adapter error A TimeoutConfig.attempt ceiling firing before the total was classified total_timeout, which is not retryable, and escaped client.get() as a bare stdlib TimeoutError because only an expired deadline became a CallError. The same except-TimeoutError branch also swallowed aiohttp's phase timeouts (its whole timeout family subclasses TimeoutError), so a sock_read timeout was labelled total_timeout and never retried. The engine now asks the cancel scope whether it fired instead of matching on the exception type. An SDK exception goes through the adapter's classifier; a fired ceiling is total_timeout when the deadline is gone and the new attempt_timeout otherwise. attempt_timeout is retryable and trips the breaker by default, like read_timeout, and a call whose last attempt died on the ceiling raises AttemptTimeoutError through the adapter translator: HttpxAttemptTimeoutError is an httpx.TimeoutException, AiohttpAttemptTimeoutError an aiohttp.ServerTimeoutError. The retry gate moved into _retry_delay in both engines so an outcome becomes final in one place. Sync adapters declare attempt_timeout collapsed into read_timeout, so the default retry config has no dead kinds under strict. Closes #24 --- clientwright/__init__.py | 2 + clientwright/adapters/_httpx_shared.py | 10 ++- clientwright/adapters/aiohttp/__init__.py | 3 + clientwright/adapters/aiohttp/capabilities.py | 1 + clientwright/adapters/aiohttp/errors.py | 15 +++- clientwright/adapters/httpx/__init__.py | 3 + clientwright/adapters/httpx/errors.py | 9 ++- clientwright/adapters/httpx2/__init__.py | 3 + clientwright/adapters/httpx2/errors.py | 9 ++- .../adapters/requests/capabilities.py | 5 ++ clientwright/adapters/urllib3/capabilities.py | 5 ++ clientwright/core/config.py | 2 + clientwright/core/engine/aio.py | 69 ++++++++++++------- clientwright/core/engine/sync.py | 47 ++++++++----- clientwright/core/errors.py | 9 +++ clientwright/core/model.py | 1 + docs/advanced/writing-an-adapter.md | 5 +- docs/agents.md | 16 +++-- docs/guide/retries.md | 7 +- docs/guide/timeouts.md | 14 ++-- docs/reference/adapters.md | 9 +-- tests/integration/adapters/test_aiohttp.py | 15 ++++ .../integration/adapters/test_httpx_async.py | 18 +++++ tests/integration/parity/test_semantics.py | 21 ++++++ tests/unit/adapters/aiohttp/test_units.py | 10 +++ tests/unit/adapters/httpx/test_units.py | 9 +++ tests/unit/core/engine/test_aio.py | 62 +++++++++++++---- tests/unit/core/test_plan.py | 9 +++ 28 files changed, 310 insertions(+), 78 deletions(-) diff --git a/clientwright/__init__.py b/clientwright/__init__.py index 44e289b..4009c3d 100644 --- a/clientwright/__init__.py +++ b/clientwright/__init__.py @@ -60,6 +60,7 @@ from .core.contracts.adapter import default_deps from .core.contracts.settings import client_config_from_settings from .core.errors import ( + AttemptTimeoutError, CallError, CircuitOpenError, ClientwrightError, @@ -117,6 +118,7 @@ def build_sync(adapter: str, config: ClientConfig, deps: AdapterDeps | None = No "UNSET", "AdapterCapabilities", "AdapterDeps", + "AttemptTimeoutError", "CallError", "CallOptions", "CallerOverride", diff --git a/clientwright/adapters/_httpx_shared.py b/clientwright/adapters/_httpx_shared.py index 3fd968d..c6c42e2 100644 --- a/clientwright/adapters/_httpx_shared.py +++ b/clientwright/adapters/_httpx_shared.py @@ -28,7 +28,7 @@ from ..core.contracts.adapter import AdapterDeps from ..core.engine.aio import AsyncAttemptEngine from ..core.engine.sync import SyncAttemptEngine -from ..core.errors import CallError, CircuitOpenError, DeadlineExceededError, TooManyRedirectsError +from ..core.errors import AttemptTimeoutError, CallError, CircuitOpenError, DeadlineExceededError, TooManyRedirectsError from ..core.model import IDEMPOTENT_METHODS, ConnMetrics, FailureKind, Outcome, RequestInfo, ResolvedTimeouts, origin_of from ..core.native import accepted_overrides, validate_native from ..core.plan import CallPlan, ClientHandle, ClientRuntime, compile_plan, register_handle @@ -105,6 +105,7 @@ def capabilities_for(adapter: str) -> AdapterCapabilities: FailureKind.READ_TIMEOUT, FailureKind.WRITE_TIMEOUT, FailureKind.POOL_TIMEOUT, + FailureKind.ATTEMPT_TIMEOUT, FailureKind.TOTAL_TIMEOUT, FailureKind.CONNECT_ERROR, FailureKind.TLS_ERROR, @@ -120,6 +121,10 @@ def capabilities_for(adapter: str) -> AdapterCapabilities: collapses={FailureKind.DNS_ERROR: FailureKind.CONNECT_ERROR}, notes={ "deadline_hard": "Hard cancellation on the async client only; the sync client clamps phases (soft).", + "attempt_timeout": ( + "Emitted by the async client only; the sync client drops the attempt ceiling, so a stall arrives as " + "read_timeout." + ), "pool_limit_per_host": "Emulated as a per-origin in-flight semaphore; limits requests, not connections.", "dns_error": f"{adapter} wraps DNS failures into ConnectError; they surface as connect_error.", "proxy_from_env": "Environment proxies are parsed into mounts; NO_PROXY entries match hosts literally.", @@ -173,6 +178,7 @@ def make_error_translator( circuit_cls: type[CircuitOpenError], deadline_cls: type[DeadlineExceededError], redirects_cls: type[TooManyRedirectsError], + attempt_cls: type[AttemptTimeoutError], ) -> Callable[[CallError], BaseException]: """Build the CallError -> dual-family translator from the bound classes.""" @@ -181,6 +187,8 @@ def translate(error: CallError) -> BaseException: return circuit_cls(error.key, error.retry_after) if isinstance(error, DeadlineExceededError): return deadline_cls(error.total) + if isinstance(error, AttemptTimeoutError): + return attempt_cls(error.attempt) if isinstance(error, TooManyRedirectsError): return redirects_cls(error.hops) return error diff --git a/clientwright/adapters/aiohttp/__init__.py b/clientwright/adapters/aiohttp/__init__.py index d449282..486f725 100644 --- a/clientwright/adapters/aiohttp/__init__.py +++ b/clientwright/adapters/aiohttp/__init__.py @@ -13,6 +13,7 @@ if TYPE_CHECKING: from .adapter import AiohttpAdapter as AiohttpAdapter from .capabilities import CAPABILITIES as CAPABILITIES + from .errors import AiohttpAttemptTimeoutError as AiohttpAttemptTimeoutError from .errors import AiohttpCircuitOpenError as AiohttpCircuitOpenError from .errors import AiohttpDeadlineExceededError as AiohttpDeadlineExceededError from .errors import AiohttpTooManyRedirectsError as AiohttpTooManyRedirectsError @@ -22,6 +23,7 @@ _EXPORTS = { "CAPABILITIES": "capabilities", "AiohttpAdapter": "adapter", + "AiohttpAttemptTimeoutError": "errors", "AiohttpCircuitOpenError": "errors", "AiohttpDeadlineExceededError": "errors", "AiohttpTooManyRedirectsError": "errors", @@ -37,6 +39,7 @@ def __getattr__(name: str) -> Any: __all__ = [ "CAPABILITIES", "AiohttpAdapter", + "AiohttpAttemptTimeoutError", "AiohttpCircuitOpenError", "AiohttpDeadlineExceededError", "AiohttpTooManyRedirectsError", diff --git a/clientwright/adapters/aiohttp/capabilities.py b/clientwright/adapters/aiohttp/capabilities.py index 6163f84..ed6ff37 100644 --- a/clientwright/adapters/aiohttp/capabilities.py +++ b/clientwright/adapters/aiohttp/capabilities.py @@ -43,6 +43,7 @@ { FailureKind.CONNECT_TIMEOUT, FailureKind.READ_TIMEOUT, + FailureKind.ATTEMPT_TIMEOUT, FailureKind.TOTAL_TIMEOUT, FailureKind.CONNECT_ERROR, FailureKind.DNS_ERROR, diff --git a/clientwright/adapters/aiohttp/errors.py b/clientwright/adapters/aiohttp/errors.py index 8aaaf92..07214bb 100644 --- a/clientwright/adapters/aiohttp/errors.py +++ b/clientwright/adapters/aiohttp/errors.py @@ -6,7 +6,13 @@ from __future__ import annotations -from ...core.errors import CallError, CircuitOpenError, DeadlineExceededError, TooManyRedirectsError +from ...core.errors import ( + AttemptTimeoutError, + CallError, + CircuitOpenError, + DeadlineExceededError, + TooManyRedirectsError, +) from ._imports import aiohttp @@ -18,6 +24,10 @@ class AiohttpDeadlineExceededError(DeadlineExceededError, aiohttp.ServerTimeoutE """Total deadline exhausted, catchable as asyncio.TimeoutError and aiohttp.ClientError.""" +class AiohttpAttemptTimeoutError(AttemptTimeoutError, aiohttp.ServerTimeoutError): + """Attempt ceiling exhausted, catchable as asyncio.TimeoutError and aiohttp.ClientError.""" + + class AiohttpTooManyRedirectsError(TooManyRedirectsError, aiohttp.TooManyRedirects): """Owned redirect limit exceeded, catchable as aiohttp.TooManyRedirects. @@ -44,12 +54,15 @@ def translate_call_error(error: CallError) -> BaseException: return AiohttpCircuitOpenError(error.key, error.retry_after) if isinstance(error, DeadlineExceededError): return AiohttpDeadlineExceededError(error.total) + if isinstance(error, AttemptTimeoutError): + return AiohttpAttemptTimeoutError(error.attempt) if isinstance(error, TooManyRedirectsError): return AiohttpTooManyRedirectsError(error.hops) return error __all__ = [ + "AiohttpAttemptTimeoutError", "AiohttpCircuitOpenError", "AiohttpDeadlineExceededError", "AiohttpTooManyRedirectsError", diff --git a/clientwright/adapters/httpx/__init__.py b/clientwright/adapters/httpx/__init__.py index 8b3c779..7a6730e 100644 --- a/clientwright/adapters/httpx/__init__.py +++ b/clientwright/adapters/httpx/__init__.py @@ -13,6 +13,7 @@ if TYPE_CHECKING: from .adapter import HttpxAdapter as HttpxAdapter from .capabilities import CAPABILITIES as CAPABILITIES + from .errors import HttpxAttemptTimeoutError as HttpxAttemptTimeoutError from .errors import HttpxCircuitOpenError as HttpxCircuitOpenError from .errors import HttpxDeadlineExceededError as HttpxDeadlineExceededError from .errors import HttpxTooManyRedirectsError as HttpxTooManyRedirectsError @@ -22,6 +23,7 @@ _EXPORTS = { "CAPABILITIES": "capabilities", "HttpxAdapter": "adapter", + "HttpxAttemptTimeoutError": "errors", "HttpxCircuitOpenError": "errors", "HttpxDeadlineExceededError": "errors", "HttpxTooManyRedirectsError": "errors", @@ -39,6 +41,7 @@ def __getattr__(name: str) -> Any: "IDEMPOTENT_EXTENSION", "ROUTE_EXTENSION", "HttpxAdapter", + "HttpxAttemptTimeoutError", "HttpxCircuitOpenError", "HttpxDeadlineExceededError", "HttpxTooManyRedirectsError", diff --git a/clientwright/adapters/httpx/errors.py b/clientwright/adapters/httpx/errors.py index ff32fbd..76e4cee 100644 --- a/clientwright/adapters/httpx/errors.py +++ b/clientwright/adapters/httpx/errors.py @@ -6,7 +6,7 @@ from __future__ import annotations -from ...core.errors import CircuitOpenError, DeadlineExceededError, TooManyRedirectsError +from ...core.errors import AttemptTimeoutError, CircuitOpenError, DeadlineExceededError, TooManyRedirectsError from .._httpx_shared import make_error_translator from ._imports import httpx @@ -19,15 +19,20 @@ class HttpxDeadlineExceededError(DeadlineExceededError, httpx.TimeoutException): """Total deadline exhausted, catchable as httpx.TimeoutException.""" +class HttpxAttemptTimeoutError(AttemptTimeoutError, httpx.TimeoutException): + """Attempt ceiling exhausted, catchable as httpx.TimeoutException.""" + + class HttpxTooManyRedirectsError(TooManyRedirectsError, httpx.TooManyRedirects): """Owned redirect limit exceeded, catchable as httpx.TooManyRedirects.""" translate_call_error = make_error_translator( - HttpxCircuitOpenError, HttpxDeadlineExceededError, HttpxTooManyRedirectsError + HttpxCircuitOpenError, HttpxDeadlineExceededError, HttpxTooManyRedirectsError, HttpxAttemptTimeoutError ) __all__ = [ + "HttpxAttemptTimeoutError", "HttpxCircuitOpenError", "HttpxDeadlineExceededError", "HttpxTooManyRedirectsError", diff --git a/clientwright/adapters/httpx2/__init__.py b/clientwright/adapters/httpx2/__init__.py index 4e7621e..1b910ef 100644 --- a/clientwright/adapters/httpx2/__init__.py +++ b/clientwright/adapters/httpx2/__init__.py @@ -14,6 +14,7 @@ if TYPE_CHECKING: from .adapter import HttpxAdapter as HttpxAdapter from .capabilities import CAPABILITIES as CAPABILITIES + from .errors import HttpxAttemptTimeoutError as HttpxAttemptTimeoutError from .errors import HttpxCircuitOpenError as HttpxCircuitOpenError from .errors import HttpxDeadlineExceededError as HttpxDeadlineExceededError from .errors import HttpxTooManyRedirectsError as HttpxTooManyRedirectsError @@ -23,6 +24,7 @@ _EXPORTS = { "CAPABILITIES": "capabilities", "HttpxAdapter": "adapter", + "HttpxAttemptTimeoutError": "errors", "HttpxCircuitOpenError": "errors", "HttpxDeadlineExceededError": "errors", "HttpxTooManyRedirectsError": "errors", @@ -40,6 +42,7 @@ def __getattr__(name: str) -> Any: "IDEMPOTENT_EXTENSION", "ROUTE_EXTENSION", "HttpxAdapter", + "HttpxAttemptTimeoutError", "HttpxCircuitOpenError", "HttpxDeadlineExceededError", "HttpxTooManyRedirectsError", diff --git a/clientwright/adapters/httpx2/errors.py b/clientwright/adapters/httpx2/errors.py index e65e866..02f76e3 100644 --- a/clientwright/adapters/httpx2/errors.py +++ b/clientwright/adapters/httpx2/errors.py @@ -7,7 +7,7 @@ from __future__ import annotations -from ...core.errors import CircuitOpenError, DeadlineExceededError, TooManyRedirectsError +from ...core.errors import AttemptTimeoutError, CircuitOpenError, DeadlineExceededError, TooManyRedirectsError from .._httpx_shared import make_error_translator from ._imports import httpx2 @@ -20,15 +20,20 @@ class HttpxDeadlineExceededError(DeadlineExceededError, httpx2.TimeoutException) """Total deadline exhausted, catchable as httpx2.TimeoutException.""" +class HttpxAttemptTimeoutError(AttemptTimeoutError, httpx2.TimeoutException): + """Attempt ceiling exhausted, catchable as httpx2.TimeoutException.""" + + class HttpxTooManyRedirectsError(TooManyRedirectsError, httpx2.TooManyRedirects): """Owned redirect limit exceeded, catchable as httpx2.TooManyRedirects.""" translate_call_error = make_error_translator( - HttpxCircuitOpenError, HttpxDeadlineExceededError, HttpxTooManyRedirectsError + HttpxCircuitOpenError, HttpxDeadlineExceededError, HttpxTooManyRedirectsError, HttpxAttemptTimeoutError ) __all__ = [ + "HttpxAttemptTimeoutError", "HttpxCircuitOpenError", "HttpxDeadlineExceededError", "HttpxTooManyRedirectsError", diff --git a/clientwright/adapters/requests/capabilities.py b/clientwright/adapters/requests/capabilities.py index 48b11a4..08a4a60 100644 --- a/clientwright/adapters/requests/capabilities.py +++ b/clientwright/adapters/requests/capabilities.py @@ -55,12 +55,17 @@ } ), collapses={ + FailureKind.ATTEMPT_TIMEOUT: FailureKind.READ_TIMEOUT, FailureKind.POOL_TIMEOUT: FailureKind.CONNECT_TIMEOUT, FailureKind.PROTOCOL_ERROR: FailureKind.DISCONNECTED, FailureKind.WRITE_TIMEOUT: FailureKind.TOTAL_TIMEOUT, }, notes={ "sync_only": "requests has no async client; build_async raises.", + "attempt_timeout": ( + "A sync runtime cannot cancel an attempt, so no ceiling ever fires; the stall it would have caught " + "arrives as the clamped read phase." + ), "no_session_timeout": ( "requests has NO session-level timeout default - a bare session.get() hangs forever. The engine " "closes that hole: every attempt is sent with the planned (connect, read) tuple." diff --git a/clientwright/adapters/urllib3/capabilities.py b/clientwright/adapters/urllib3/capabilities.py index 9d41b03..c3cf953 100644 --- a/clientwright/adapters/urllib3/capabilities.py +++ b/clientwright/adapters/urllib3/capabilities.py @@ -56,11 +56,16 @@ } ), collapses={ + FailureKind.ATTEMPT_TIMEOUT: FailureKind.READ_TIMEOUT, FailureKind.PROTOCOL_ERROR: FailureKind.DISCONNECTED, FailureKind.WRITE_TIMEOUT: FailureKind.TOTAL_TIMEOUT, }, notes={ "sync_only": "urllib3 has no async client; build_async raises.", + "attempt_timeout": ( + "A sync runtime cannot cancel an attempt, so no ceiling ever fires; the stall it would have caught " + "arrives as the clamped read phase." + ), "seam": ( "The engine is injected as an INSTANCE urlopen on a genuine PoolManager (type(client) is " "urllib3.PoolManager); recursive native redirect hops re-enter it and pass straight through." diff --git a/clientwright/core/config.py b/clientwright/core/config.py index f23a281..c1d3be0 100644 --- a/clientwright/core/config.py +++ b/clientwright/core/config.py @@ -87,6 +87,7 @@ class RetryMode(StrEnum): FailureKind.DNS_ERROR, FailureKind.POOL_TIMEOUT, FailureKind.READ_TIMEOUT, + FailureKind.ATTEMPT_TIMEOUT, FailureKind.DISCONNECTED, } ) @@ -99,6 +100,7 @@ class RetryMode(StrEnum): FailureKind.READ_TIMEOUT, FailureKind.WRITE_TIMEOUT, FailureKind.POOL_TIMEOUT, + FailureKind.ATTEMPT_TIMEOUT, FailureKind.TOTAL_TIMEOUT, FailureKind.CONNECT_ERROR, FailureKind.DNS_ERROR, diff --git a/clientwright/core/engine/aio.py b/clientwright/core/engine/aio.py index e0b5104..427c8b3 100644 --- a/clientwright/core/engine/aio.py +++ b/clientwright/core/engine/aio.py @@ -15,8 +15,8 @@ from ..contracts.adapter import AdapterDeps from ..contracts.message import AsyncNormalizer, RequestView, ResponseView -from ..errors import CallError, CircuitOpenError, DeadlineExceededError, TooManyRedirectsError -from ..model import Attempt, FailureKind, Outcome +from ..errors import AttemptTimeoutError, CallError, CircuitOpenError, DeadlineExceededError, TooManyRedirectsError +from ..model import Attempt, FailureKind, Outcome, RequestInfo from ..plan import CallPlan, ClientRuntime from ..policy.budget import Deadline from ..telemetry.emitter import CallObservation, ClientTelemetry @@ -79,6 +79,8 @@ def _call_error_outcome(self, error: CallError) -> Outcome: return Outcome(kind=FailureKind.CIRCUIT_OPEN, exception=error) if isinstance(error, DeadlineExceededError): return Outcome(kind=FailureKind.TOTAL_TIMEOUT, exception=error) + if isinstance(error, AttemptTimeoutError): + return Outcome(kind=FailureKind.ATTEMPT_TIMEOUT, exception=error) return Outcome(kind=FailureKind.UNKNOWN, exception=error) def _wrap_stream(self, response: ResponseView, info: Any) -> None: @@ -171,6 +173,30 @@ async def _freeze_if_needed(self, request: RequestView) -> bool: return True return await self._norm.freeze(request) + def _retry_delay( + self, info: RequestInfo, history: list[Attempt], deadline: Deadline, replayable: bool + ) -> float | None: + """Backoff before the next attempt, or None when the last outcome is final.""" + plan = self._plan + runtime = self._runtime + if plan.retry_policy is None: + return None + decision = plan.retry_policy.decide( + info=info, + history=history, + remaining=deadline.remaining(), + replayable=replayable, + rng=runtime.rng, + ) + if not decision.retry: + if decision.reason in SKIP_REASONS: + self._telemetry.retry_skipped(decision.reason) + return None + if runtime.retry_budgets is not None and not runtime.retry_budgets.try_spend(info.origin): + self._telemetry.retry_skipped("budget") + return None + return decision.delay + async def _attempts( self, request: RequestView, @@ -195,8 +221,9 @@ async def _attempts( attempt_started = runtime.clock() observation.attempts += 1 response: ResponseView | None = None + ceiling = asyncio.timeout(timeouts.attempt) try: - async with asyncio.timeout(timeouts.attempt): + async with ceiling: native_response = await send(request) response = self._norm.wrap_response(native_response) outcome = self._norm.classify_response(response) @@ -204,10 +231,17 @@ async def _attempts( raise except asyncio.CancelledError: raise - except TimeoutError as error: - outcome = Outcome(kind=FailureKind.TOTAL_TIMEOUT, exception=error) except Exception as error: - outcome = Outcome(kind=self._norm.classify_error(error), exception=error) + # Only a fired ceiling is the engine's own timeout. Matching on + # TimeoutError would also swallow the SDK's: aiohttp's whole + # timeout family subclasses it. + if not ceiling.expired(): + kind = self._norm.classify_error(error) + elif deadline.expired: + kind = FailureKind.TOTAL_TIMEOUT + else: + kind = FailureKind.ATTEMPT_TIMEOUT + outcome = Outcome(kind=kind, exception=error) attempt = Attempt( index=len(history) + 1, started=attempt_started, @@ -221,27 +255,16 @@ async def _attempts( self._telemetry.attempt_end(observation, info, attempt) if outcome.kind is FailureKind.TOTAL_TIMEOUT and deadline.expired: raise DeadlineExceededError(deadline.total or 0.0) from outcome.exception - if plan.retry_policy is None: - return response, outcome - decision = plan.retry_policy.decide( - info=info, - history=history, - remaining=deadline.remaining(), - replayable=replayable, - rng=runtime.rng, - ) - if not decision.retry: - if decision.reason in SKIP_REASONS: - self._telemetry.retry_skipped(decision.reason) - return response, outcome - if runtime.retry_budgets is not None and not runtime.retry_budgets.try_spend(info.origin): - self._telemetry.retry_skipped("budget") + delay = self._retry_delay(info, history, deadline, replayable) + if delay is None: + if outcome.kind is FailureKind.ATTEMPT_TIMEOUT: + raise AttemptTimeoutError(timeouts.attempt or 0.0) from outcome.exception return response, outcome if response is not None: await self._norm.discard(response) await self._norm.rewind(request) - if decision.delay > 0: - await asyncio.sleep(decision.delay) + if delay > 0: + await asyncio.sleep(delay) __all__ = ["AsyncAttemptEngine", "AsyncSend", "ErrorTranslator"] diff --git a/clientwright/core/engine/sync.py b/clientwright/core/engine/sync.py index fa7a6d6..e669185 100644 --- a/clientwright/core/engine/sync.py +++ b/clientwright/core/engine/sync.py @@ -17,7 +17,7 @@ from ..contracts.adapter import AdapterDeps from ..contracts.message import RequestView, ResponseView, SyncNormalizer from ..errors import CallError, CircuitOpenError, DeadlineExceededError, TooManyRedirectsError -from ..model import Attempt, FailureKind, Outcome +from ..model import Attempt, FailureKind, Outcome, RequestInfo from ..plan import CallPlan, ClientRuntime from ..policy.budget import Deadline from ..telemetry.emitter import CallObservation, ClientTelemetry @@ -169,6 +169,30 @@ def _freeze_if_needed(self, request: RequestView) -> bool: return True return self._norm.freeze(request) + def _retry_delay( + self, info: RequestInfo, history: list[Attempt], deadline: Deadline, replayable: bool + ) -> float | None: + """Backoff before the next attempt, or None when the last outcome is final.""" + plan = self._plan + runtime = self._runtime + if plan.retry_policy is None: + return None + decision = plan.retry_policy.decide( + info=info, + history=history, + remaining=deadline.remaining(), + replayable=replayable, + rng=runtime.rng, + ) + if not decision.retry: + if decision.reason in SKIP_REASONS: + self._telemetry.retry_skipped(decision.reason) + return None + if runtime.retry_budgets is not None and not runtime.retry_budgets.try_spend(info.origin): + self._telemetry.retry_skipped("budget") + return None + return decision.delay + def _attempts( self, request: RequestView, @@ -216,27 +240,14 @@ def _attempts( self._telemetry.attempt_end(observation, info, attempt) if outcome.kind is FailureKind.TOTAL_TIMEOUT and deadline.expired: raise DeadlineExceededError(deadline.total or 0.0) from outcome.exception - if plan.retry_policy is None: - return response, outcome - decision = plan.retry_policy.decide( - info=info, - history=history, - remaining=deadline.remaining(), - replayable=replayable, - rng=runtime.rng, - ) - if not decision.retry: - if decision.reason in SKIP_REASONS: - self._telemetry.retry_skipped(decision.reason) - return response, outcome - if runtime.retry_budgets is not None and not runtime.retry_budgets.try_spend(info.origin): - self._telemetry.retry_skipped("budget") + delay = self._retry_delay(info, history, deadline, replayable) + if delay is None: return response, outcome if response is not None: self._norm.discard(response) self._norm.rewind(request) - if decision.delay > 0: - time.sleep(decision.delay) + if delay > 0: + time.sleep(delay) __all__ = ["SyncAttemptEngine", "SyncSend"] diff --git a/clientwright/core/errors.py b/clientwright/core/errors.py index 6068f19..7b22b1a 100644 --- a/clientwright/core/errors.py +++ b/clientwright/core/errors.py @@ -86,6 +86,14 @@ def __init__(self, total: float) -> None: self.total = total +class AttemptTimeoutError(CallError): + """The last attempt hit its ceiling while the total deadline still had room.""" + + def __init__(self, attempt: float) -> None: + super().__init__(f"Attempt ceiling of {attempt:.3f}s exhausted") + self.attempt = attempt + + class TooManyRedirectsError(CallError): def __init__(self, hops: int) -> None: super().__init__(f"Exceeded {hops} redirect hops") @@ -105,6 +113,7 @@ class NotReplayableError(CallError): __all__ = [ + "AttemptTimeoutError", "CallError", "CircuitOpenError", "ClientwrightError", diff --git a/clientwright/core/model.py b/clientwright/core/model.py index 1f757ba..537e426 100644 --- a/clientwright/core/model.py +++ b/clientwright/core/model.py @@ -24,6 +24,7 @@ class FailureKind(StrEnum): READ_TIMEOUT = "read_timeout" WRITE_TIMEOUT = "write_timeout" POOL_TIMEOUT = "pool_timeout" + ATTEMPT_TIMEOUT = "attempt_timeout" TOTAL_TIMEOUT = "total_timeout" CONNECT_ERROR = "connect_error" DNS_ERROR = "dns_error" diff --git a/docs/advanced/writing-an-adapter.md b/docs/advanced/writing-an-adapter.md index 0c3734c..831c4a2 100644 --- a/docs/advanced/writing-an-adapter.md +++ b/docs/advanced/writing-an-adapter.md @@ -66,7 +66,10 @@ CAPABILITIES = AdapterCapabilities( This module must import **without the SDK installed** — it is what `capabilities_matrix()` shows to users deciding whether to adopt you. Understate rather than overstate: `dropped` with a reason beats a knob that silently does -nothing. +nothing. `emits` also covers what the engine itself surfaces through your seam: +`total_timeout`, `circuit_open`, `cancelled` and, on an async adapter, +`attempt_timeout` — a default retryable kind you neither emit nor collapse is +reported dead on every build. ## 5. Register and test diff --git a/docs/agents.md b/docs/agents.md index 404baa2..220173c 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -210,7 +210,7 @@ the circuit registry, the retry budget registry and the per-origin limiters from | Field | Default | Meaning | |---|---|---| | `total` | `30.0` | wall clock for the whole logical call: every attempt, backoff sleep and redirect hop. `None` = unbounded | -| `attempt` | `UNSET` | ceiling for one attempt. Async enforces it by cancellation; sync cannot and reports it dropped | +| `attempt` | `UNSET` | ceiling for one attempt. Async enforces it by cancellation and labels a cut attempt `attempt_timeout` (retryable by default); sync cannot and reports it dropped | | `connect` | `5.0` | | | `read` | `UNSET` | | | `write` | `UNSET` | | @@ -240,7 +240,7 @@ each attempt — a phase that is `None` after resolution becomes exactly the rem | `max_backoff` | `10.0` | | | `multiplier` | `2.0` | `>= 1` | | `jitter` | `0.2` | within `[0, 1]`; multiplicative ±20 % | -| `retryable_kinds` | `DEFAULT_RETRYABLE_KINDS` | `connect_timeout`, `connect_error`, `dns_error`, `pool_timeout`, `read_timeout`, `disconnected` | +| `retryable_kinds` | `DEFAULT_RETRYABLE_KINDS` | `connect_timeout`, `connect_error`, `dns_error`, `pool_timeout`, `read_timeout`, `attempt_timeout`, `disconnected` | | `retryable_status` | `DEFAULT_RETRYABLE_STATUS` | `{429, 502, 503, 504}` — note `500` is absent | | `methods` | `IDEMPOTENT_METHODS` | `{GET, HEAD, PUT, DELETE, OPTIONS, TRACE}` | | `respect_retry_after` | `True` | a `Retry-After` (seconds or HTTP-date) replaces the computed backoff | @@ -285,7 +285,7 @@ exports. | Name | Values / fields | |---|---| -| `FailureKind` | `connect_timeout`, `read_timeout`, `write_timeout`, `pool_timeout`, `total_timeout`, `connect_error`, `dns_error`, `tls_error`, `protocol_error`, `disconnected`, `body_error`, `status`, `cancelled`, `circuit_open`, `unknown` | +| `FailureKind` | `connect_timeout`, `read_timeout`, `write_timeout`, `pool_timeout`, `attempt_timeout`, `total_timeout`, `connect_error`, `dns_error`, `tls_error`, `protocol_error`, `disconnected`, `body_error`, `status`, `cancelled`, `circuit_open`, `unknown` | | `Outcome` | `kind` (`None` means success), `status_code`, `retry_after`, `exception`; `.ok` | | `RequestInfo` | `method`, `origin`, `url`, `route`, `idempotent`; `.circuit_key(mode)` | | `ResolvedTimeouts` | `connect`, `read`, `write`, `pool_acquire`, `attempt` | @@ -604,14 +604,16 @@ a class that *also* inherits the SDK's own error family, so an existing | `NativeConfigError` | base of the passthrough errors: `UnknownNativeSlotError`, `ReservedNativeKeyError`, `UnknownNativeKeyError` (with a did-you-mean), `NativeConfigConflictError` — the four subclasses live in `clientwright.core.errors` | | `CircuitOpenError` | the circuit for this key is open; `.key`, `.retry_after` | | `DeadlineExceededError` | the total deadline is exhausted; `.total` | +| `AttemptTimeoutError` | the last attempt hit the `attempt` ceiling while the total still had room; `.attempt`. Async engines only | | `TooManyRedirectsError` | more than `max_redirects` hops; `.hops` | | `NotReplayableError` | exported, and passed through the adapter translators unchanged, but the engine never raises it: a non-replayable body ends the call with the response it already has plus a `retry_skipped{reason="non_replayable"}` counter | | `CallerOverrideForbiddenError` | a per-call timeout under `CallerOverride.RAISE`; a `CallError`, importable from `clientwright.core.policy.timeout` | -Per-adapter classes are the same three names with the adapter's prefix: -`HttpxCircuitOpenError`, `HttpxDeadlineExceededError`, `HttpxTooManyRedirectsError` (also -under `clientwright.adapters.httpx2`, deliberately with the same class names), and the -`Aiohttp*`, `Requests*`, `Urllib3*` trios. +Per-adapter classes are the same names with the adapter's prefix: +`HttpxCircuitOpenError`, `HttpxDeadlineExceededError`, `HttpxAttemptTimeoutError`, +`HttpxTooManyRedirectsError` (also under `clientwright.adapters.httpx2`, deliberately with +the same class names) and the `Aiohttp*` four; the `Requests*` and `Urllib3*` trios have no +`AttemptTimeoutError`, because a sync engine never raises one. ## Documentation map diff --git a/docs/guide/retries.md b/docs/guide/retries.md index f433e74..843ca22 100644 --- a/docs/guide/retries.md +++ b/docs/guide/retries.md @@ -34,9 +34,10 @@ Two lists decide, and both are yours to change: `500` is treated as "the server executed something and failed" — replaying it is a decision you must opt into, not a default. - **Failure kinds**: `connect_timeout`, `connect_error`, `dns_error`, - `pool_timeout`, `read_timeout`, `disconnected` — infrastructure failures where - the request plausibly never ran. A `read_timeout` *after* bytes were sent is the - riskiest of these, which is exactly why the idempotency gate below exists. + `pool_timeout`, `read_timeout`, `attempt_timeout`, `disconnected` — + infrastructure failures where the request plausibly never ran. A `read_timeout` + or `attempt_timeout` *after* bytes were sent is the riskiest of these, which is + exactly why the idempotency gate below exists. ```python from clientwright import FailureKind diff --git a/docs/guide/timeouts.md b/docs/guide/timeouts.md index 39a922c..a22ac0c 100644 --- a/docs/guide/timeouts.md +++ b/docs/guide/timeouts.md @@ -47,7 +47,12 @@ translate to whatever the SDK natively understands. Two things are worth knowing `on_unsupported="strict"`. `attempt` is the odd one out: a ceiling for one whole attempt regardless of phase. -Async engines enforce it by cancellation; sync engines by clamping phases. +Async engines enforce it by cancellation, and an attempt cut by it ends with the +outcome `attempt_timeout`, which is retryable by default — `attempt=0.4` under +`total=1.0` cuts a hung attempt at 0.4 s and tries again while the budget lasts. A +ceiling clamped to what the total has left is the total firing, and is labelled +`total_timeout`. Sync engines cannot cancel a blocked attempt and report the knob +dropped. ## When the caller also passes a timeout @@ -96,6 +101,7 @@ deps = AdapterDeps(deadline_source=my_source) # (2)! A call that dies on the total raises `DeadlineExceededError` — dual-inherited from the adapter's native error family, so your existing `except httpx.TimeoutException` -keeps catching it — and lands in metrics with `outcome="total_timeout"`. A phase -that fired first keeps its own name (`connect_timeout`, `read_timeout`, ...); the -taxonomy never merges them. +keeps catching it — and lands in metrics with `outcome="total_timeout"`. A call +whose last attempt died on the `attempt` ceiling raises `AttemptTimeoutError` the +same way, with `outcome="attempt_timeout"`. A phase that fired first keeps its own +name (`connect_timeout`, `read_timeout`, ...); the taxonomy never merges them. diff --git a/docs/reference/adapters.md b/docs/reference/adapters.md index 602b342..e1df40b 100644 --- a/docs/reference/adapters.md +++ b/docs/reference/adapters.md @@ -11,14 +11,15 @@ this page is the complete list. ## Per-adapter exports Every adapter package exports its adapter class, its per-call channel, and its -dual-family error trio (each inherits both the clientwright error and the SDK's -native family): +dual-family errors (each inherits both the clientwright error and the SDK's +native family; only the async adapters carry an `AttemptTimeoutError`, because a +sync engine never raises one): | Package | Per-call channel | Errors | |---|---|---| -| `clientwright.adapters.httpx` | `ROUTE_EXTENSION`, `IDEMPOTENT_EXTENSION` (request extensions) | `HttpxCircuitOpenError`, `HttpxDeadlineExceededError`, `HttpxTooManyRedirectsError` | +| `clientwright.adapters.httpx` | `ROUTE_EXTENSION`, `IDEMPOTENT_EXTENSION` (request extensions) | `HttpxCircuitOpenError`, `HttpxDeadlineExceededError`, `HttpxAttemptTimeoutError`, `HttpxTooManyRedirectsError` | | `clientwright.adapters.httpx2` | same names as httpx | same class names as httpx, inheriting `httpx2`'s family | -| `clientwright.adapters.aiohttp` | `call_options(route=..., idempotent=...)` | `AiohttpCircuitOpenError`, `AiohttpDeadlineExceededError`, `AiohttpTooManyRedirectsError` | +| `clientwright.adapters.aiohttp` | `call_options(route=..., idempotent=...)` | `AiohttpCircuitOpenError`, `AiohttpDeadlineExceededError`, `AiohttpAttemptTimeoutError`, `AiohttpTooManyRedirectsError` | | `clientwright.adapters.requests` | `call_options(...)` | `RequestsCircuitOpenError`, `RequestsDeadlineExceededError`, `RequestsTooManyRedirectsError` | | `clientwright.adapters.urllib3` | `call_options(...)` | `Urllib3CircuitOpenError`, `Urllib3DeadlineExceededError`, `Urllib3TooManyRedirectsError` | diff --git a/tests/integration/adapters/test_aiohttp.py b/tests/integration/adapters/test_aiohttp.py index d1353b7..68f4c09 100644 --- a/tests/integration/adapters/test_aiohttp.py +++ b/tests/integration/adapters/test_aiohttp.py @@ -175,6 +175,21 @@ async def test__slow_response__total_deadline_cancels_and_translates(origin: Ori await client.close() +async def test__sock_read_timeout__classified_read_timeout_and_retried( + origin: OriginServer, metrics: RecordingMetrics, deps: AdapterDeps +) -> None: + # aiohttp's timeout family subclasses TimeoutError; the engine must leave + # it to the adapter's classifier instead of calling it the total. + config = base_config(origin, timeout=TimeoutConfig(total=5.0, connect=1.0, read=0.2), retry=FAST_RETRY) + client = await build(config, deps) + with pytest.raises(aiohttp.ServerTimeoutError): + await client.get("/slow/1") + await client.close() + assert origin.request_count("/slow/1") == 3 + assert [record["outcome"] for record in metrics.attempts] == ["read_timeout"] * 3 + assert metrics.calls[0]["outcome"] == "read_timeout" + + async def test__deadline_header__stamped_with_remaining_budget(origin: OriginServer, deps: AdapterDeps) -> None: config = base_config(origin, timeout=TimeoutConfig(total=30.0), deadline_header="X-Deadline-Ms") client = await build(config, deps) diff --git a/tests/integration/adapters/test_httpx_async.py b/tests/integration/adapters/test_httpx_async.py index b8c12b6..edb6f90 100644 --- a/tests/integration/adapters/test_httpx_async.py +++ b/tests/integration/adapters/test_httpx_async.py @@ -14,6 +14,7 @@ IDEMPOTENT_EXTENSION, ROUTE_EXTENSION, HttpxCircuitOpenError, + HttpxDeadlineExceededError, HttpxTooManyRedirectsError, ) from clientwright.core.config import CircuitBreakerConfig # noqa: E402 @@ -166,6 +167,23 @@ async def test__slow_body__total_deadline_cancels_and_translates(origin: OriginS await client.aclose() +async def test__attempt_ceiling_inside_the_total__retried_until_the_deadline( + origin: OriginServer, metrics: RecordingMetrics, deps: AdapterDeps +) -> None: + config = base_config(origin, timeout=TimeoutConfig(total=1.0, attempt=0.4), retry=FAST_RETRY) + client = await build(config, deps) + started = asyncio.get_running_loop().time() + with pytest.raises(httpx.TimeoutException) as excinfo: + await client.get("/slow/3") + elapsed = asyncio.get_running_loop().time() - started + assert isinstance(excinfo.value, HttpxDeadlineExceededError) # the third ceiling is clamped to what is left + assert 0.9 < elapsed < 1.5 + assert origin.request_count("/slow/3") == 3 + assert [record["outcome"] for record in metrics.attempts] == ["attempt_timeout", "attempt_timeout", "total_timeout"] + assert metrics.calls[0]["outcome"] == "total_timeout" + await client.aclose() + + async def test__deadline_header__stamped_with_remaining_budget(origin: OriginServer, deps: AdapterDeps) -> None: config = base_config(origin, timeout=TimeoutConfig(total=30.0), deadline_header="X-Deadline-Ms") client = await build(config, deps) diff --git a/tests/integration/parity/test_semantics.py b/tests/integration/parity/test_semantics.py index 61581c2..875130d 100644 --- a/tests/integration/parity/test_semantics.py +++ b/tests/integration/parity/test_semantics.py @@ -258,6 +258,27 @@ def test__total_deadline__caps_slow_origin_sync(adapter_name: str, origin: Origi assert metrics.calls[0]["outcome"] == "read_timeout" +# --- attempt ceiling: cut, retried, then the family's own timeout error ------ + + +@pytest.mark.parametrize("adapter_name", adapter_params(ASYNC_ADAPTERS)) +async def test__attempt_ceiling__retried_then_family_timeout(adapter_name: str, origin: OriginServer) -> None: + driver = get_driver(adapter_name) + metrics, deps = fresh_deps() + timeout = TimeoutConfig(total=5.0, attempt=0.3, connect=1.0) + client = driver.build(battery_config(driver, origin, retry=FAST_RETRY, timeout=timeout), deps) + started = time.monotonic() + try: + with pytest.raises(driver.family_errors()): + await driver.request(client, origin.url, "GET", "/slow/3") + finally: + await driver.close(client) + assert time.monotonic() - started < 2.5 # three ceilings and two backoffs, never a 3s stall + assert origin.request_count("/slow/3") == 3 + assert [record["outcome"] for record in metrics.attempts] == ["attempt_timeout"] * 3 + assert metrics.calls[0]["outcome"] == "attempt_timeout" + + # --- owned redirects: hops inside ONE logical call --------------------------- diff --git a/tests/unit/adapters/aiohttp/test_units.py b/tests/unit/adapters/aiohttp/test_units.py index 3a02d9e..bd619e8 100644 --- a/tests/unit/adapters/aiohttp/test_units.py +++ b/tests/unit/adapters/aiohttp/test_units.py @@ -15,6 +15,7 @@ import clientwright # noqa: E402 from clientwright.adapters.aiohttp import ( # noqa: E402 AiohttpAdapter, + AiohttpAttemptTimeoutError, AiohttpCircuitOpenError, AiohttpDeadlineExceededError, AiohttpTooManyRedirectsError, @@ -26,6 +27,7 @@ from clientwright.adapters.aiohttp.options import current_call_options # noqa: E402 from clientwright.adapters.aiohttp.views import AiohttpRequestView # noqa: E402 from clientwright.core.errors import ( # noqa: E402 + AttemptTimeoutError, CallError, CircuitOpenError, DeadlineExceededError, @@ -129,6 +131,14 @@ def test__deadline__catchable_as_timeout_and_client_error() -> None: assert "2.000" in str(error) +def test__attempt_timeout__catchable_as_timeout_and_client_error() -> None: + error = translate_call_error(AttemptTimeoutError(0.4)) + assert isinstance(error, AiohttpAttemptTimeoutError) + assert isinstance(error, TimeoutError) + assert isinstance(error, aiohttp.ClientError) + assert "0.400" in str(error) + + def test__too_many_redirects__catchable_and_printable() -> None: error = translate_call_error(TooManyRedirectsError(5)) assert isinstance(error, AiohttpTooManyRedirectsError) diff --git a/tests/unit/adapters/httpx/test_units.py b/tests/unit/adapters/httpx/test_units.py index cb49406..5052070 100644 --- a/tests/unit/adapters/httpx/test_units.py +++ b/tests/unit/adapters/httpx/test_units.py @@ -17,6 +17,7 @@ IDEMPOTENT_EXTENSION, ROUTE_EXTENSION, HttpxAdapter, + HttpxAttemptTimeoutError, HttpxCircuitOpenError, HttpxDeadlineExceededError, HttpxTooManyRedirectsError, @@ -28,6 +29,7 @@ from clientwright.core.capabilities import Capability # noqa: E402 from clientwright.core.config import ClientConfig, NativeOptions, TlsConfig # noqa: E402 from clientwright.core.errors import ( # noqa: E402 + AttemptTimeoutError, CircuitOpenError, DeadlineExceededError, NotReplayableError, @@ -170,6 +172,13 @@ def test__deadline__catchable_as_httpx_timeout() -> None: assert isinstance(error, httpx.TimeoutException) +def test__attempt_timeout__catchable_as_httpx_timeout() -> None: + error = translate_call_error(AttemptTimeoutError(0.4)) + assert isinstance(error, HttpxAttemptTimeoutError) + assert isinstance(error, httpx.TimeoutException) + assert error.attempt == 0.4 + + def test__redirects__catchable_as_httpx_too_many_redirects() -> None: error = translate_call_error(TooManyRedirectsError(5)) assert isinstance(error, HttpxTooManyRedirectsError) diff --git a/tests/unit/core/engine/test_aio.py b/tests/unit/core/engine/test_aio.py index 6552096..924374a 100644 --- a/tests/unit/core/engine/test_aio.py +++ b/tests/unit/core/engine/test_aio.py @@ -18,7 +18,12 @@ TimeoutConfig, ) from clientwright.core.contracts.adapter import AdapterDeps -from clientwright.core.errors import CircuitOpenError, DeadlineExceededError, TooManyRedirectsError +from clientwright.core.errors import ( + AttemptTimeoutError, + CircuitOpenError, + DeadlineExceededError, + TooManyRedirectsError, +) from clientwright.core.model import ConnMetrics, ResolvedTimeouts from clientwright.core.policy.circuit import CircuitState from clientwright.core.testing import ManualClock @@ -27,6 +32,7 @@ FixedDeadlineSource, Harness, ScriptedSend, + Slow, as_async_send, fast_retry, make_config, @@ -101,18 +107,38 @@ async def test__ambient_budget_pre_expired__raises_before_any_send() -> None: assert harness.call_outcomes == ["total_timeout"] -async def test__attempt_timeout_with_budget_left__surfaces_the_native_error() -> None: - # The deadline clock is manual and never advances: the attempt timed out - # while the total budget still has room, so the SDK error must surface. - config = make_config(retry=None, timeout=TimeoutConfig(total=30.0, attempt=0.05)) +async def test__attempt_ceiling_with_budget_left__retried_then_attempt_timeout_error() -> None: + # The deadline clock is manual and never advances: every ceiling fires with + # the whole total still available, so each one is a retryable attempt. + config = make_config(timeout=TimeoutConfig(total=30.0, attempt=0.05)) harness = Harness(config, clock=ManualClock()) + stall = Slow(1.0, FakeResponse(200)) + script = ScriptedSend(stall, stall, stall) + with pytest.raises(AttemptTimeoutError) as excinfo: + await harness.engine.run(EngineRequest(), as_async_send(script)) + assert excinfo.value.attempt == 0.05 + assert script.sent == 3 + assert harness.attempt_outcomes == ["attempt_timeout"] * 3 + assert harness.call_outcomes == ["attempt_timeout"] - async def timed_out_send(request: EngineRequest) -> FakeResponse: - raise TimeoutError("attempt ceiling") - with pytest.raises(TimeoutError): - await harness.engine.run(EngineRequest(), timed_out_send) - assert harness.call_outcomes == ["total_timeout"] # classified, not swallowed +async def test__attempt_ceiling_without_a_total__attempt_timeout_error() -> None: + config = make_config(retry=None, timeout=TimeoutConfig(total=None, attempt=0.05)) + harness = Harness(config) + script = ScriptedSend(Slow(1.0, FakeResponse(200))) + with pytest.raises(AttemptTimeoutError): + await harness.engine.run(EngineRequest(), as_async_send(script)) + assert harness.call_outcomes == ["attempt_timeout"] + + +async def test__sdk_timeout_subclass__classified_by_the_adapter_not_as_the_ceiling() -> None: + # aiohttp's whole timeout family subclasses TimeoutError: without a fired + # ceiling the exception is the SDK's and keeps the adapter's classification. + harness = Harness(make_config(timeout=TimeoutConfig(total=30.0, attempt=5.0))) + script = ScriptedSend(TimeoutError("sock_read"), FakeResponse(200)) + result = await harness.engine.run(EngineRequest(), as_async_send(script)) + assert result.status_code == 200 + assert harness.attempt_outcomes == ["read_timeout", "success"] # the fake normalizer's mapping, retried async def test__total_deadline_exhausted_mid_attempt__deadline_error() -> None: @@ -121,8 +147,9 @@ async def test__total_deadline_exhausted_mid_attempt__deadline_error() -> None: harness = Harness(config, clock=clock) async def stalled_send(request: EngineRequest) -> FakeResponse: - clock.advance(0.1) # the attempt burned through the whole total budget - raise TimeoutError("attempt ceiling") + clock.advance(0.1) # the attempt burns through the whole total budget... + await asyncio.sleep(1.0) # ...and the ceiling, clamped to it, cuts the attempt + return FakeResponse(200) with pytest.raises(DeadlineExceededError): await harness.engine.run(EngineRequest(), stalled_send) @@ -182,6 +209,17 @@ async def test__call_error_from_below__classified_and_trips_the_breaker() -> Non await harness.engine.run(EngineRequest(), as_async_send(ScriptedSend())) +async def test__attempt_timeout_error_from_below__classified_and_trips_the_breaker() -> None: + config = make_config(retry=None, circuit_breaker=_breaker_config()) + harness = Harness(config) + script = ScriptedSend(AttemptTimeoutError(0.5)) + with pytest.raises(AttemptTimeoutError): + await harness.engine.run(EngineRequest(), as_async_send(script)) + assert harness.call_outcomes == ["attempt_timeout"] + with pytest.raises(CircuitOpenError): + await harness.engine.run(EngineRequest(), as_async_send(ScriptedSend())) + + # --- retry gates --- diff --git a/tests/unit/core/test_plan.py b/tests/unit/core/test_plan.py index 4bcd310..38b443f 100644 --- a/tests/unit/core/test_plan.py +++ b/tests/unit/core/test_plan.py @@ -13,6 +13,7 @@ DurationBoundary, SeamGranularity, Support, + capabilities_matrix, dead_retryable_kinds, ) from clientwright.core.config import ClientConfig, PoolConfig, RetryConfig, RetryMode, UnsupportedPolicy @@ -107,6 +108,14 @@ def test__dead_retryable_kinds__reported_for_unreachable_kinds() -> None: assert FailureKind.CONNECT_ERROR not in plan.report.dead_retryable_kinds +def test__default_retry_kinds__reachable_on_every_registered_adapter() -> None: + # A default retryable kind an adapter can never produce would turn every + # strict build of a default config into a failed deploy. + for capabilities in capabilities_matrix().values(): + plan = compile_plan(ClientConfig(service_name="svc"), capabilities, native_timeout_defaults=NATIVE_DEFAULTS) + assert plan.report.dead_retryable_kinds == frozenset(), capabilities.adapter + + # --- dead retryable kinds ---