diff --git a/docs/agents.md b/docs/agents.md index f36d670..90448a2 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -106,8 +106,9 @@ async def main() -> None: asyncio.run(main()) ``` -A hand-built chain, which is what you need for per-method budgets, request-budget -propagation or wait-for-ready: +A hand-built chain, which is what you need for per-method budgets — request-budget +propagation and wait-for-ready are reachable from a settings object too, through their +own optional blocks: ```python from grpc_client_kit import ( @@ -298,7 +299,9 @@ Settings and collaborator protocols, all `runtime_checkable` and all exported: `RetrySettingsProtocol`, `CircuitBreakerSettingsProtocol`, `LoadBalancerSettingsProtocol`, `HealthCheckerSettingsProtocol`, `ChannelProviderProtocol`, `HealthCheckerProtocol`, `HealthStatusCallbackProtocol`, `GrpcClientMetricsProtocol`, `RetryMetricsProtocol`, -`CircuitBreakerMetricsProtocol`. Also `metadata_to_dict(metadata)` and `__version__`. +`CircuitBreakerMetricsProtocol`, and the three that describe the optional settings blocks — +`GrpcChannelExtrasProtocol`, `GrpcObservabilityExtrasProtocol`, +`FullGrpcClientSettingsProtocol`. Also `metadata_to_dict(metadata)` and `__version__`. Names that exist but are **not** re-exported at package level — import them from the module named beside them: @@ -312,11 +315,12 @@ named beside them: | `AsyncPassiveOutlierInterceptor`, `DEFAULT_QUARANTINE_SECONDS` | `grpc_client_kit.interceptors.outlier` | | `validate_target`, `MIN_PORT`, `MAX_PORT` | `grpc_client_kit.validation` | | `create_aio_channel` | `grpc_client_kit.utils` | -| `GrpcChannelExtrasProtocol`, `GrpcObservabilityExtrasProtocol`, `FullGrpcClientSettingsProtocol` | `grpc_client_kit.protocols` | -`ChannelWrapper` and `chain_token` in `grpc_client_kit.channel` are pool internals, left out -of the public surface on purpose: exporting them would freeze the pool's implementation into -the compatibility contract. Do not build on them. +`ChannelWrapper` and `chain_token` in `grpc_client_kit.channel`, and `MethodCircuitState` in +`grpc_client_kit.interceptors.circuit_breaker`, are internals left out of the public surface on +purpose: exporting them would freeze those implementations into the compatibility contract. Do +not build on them — a breaker's state is read through `get_states()`, which returns +`CircuitBreakerStatus` snapshots. ### Settings objects @@ -423,7 +427,11 @@ Per-method budgets are the one thing settings cannot express: `timeout` carries both. Native retries *without* kit retries are fully supported. 19. **Batteries are opt-in, and a missing one is a warning, not an error.** `import grpc_client_kit` never reaches for an extra. `HealthChecker` resolves on first - attribute access and raises `ImportError` naming `[health]`. Tracing, metrics and the + attribute access and raises `ImportError` naming `[health]` — an `ImportError` and not an + `AttributeError`, so a broken install says so rather than looking like a name that never + existed. The cost is that `hasattr(grpc_client_kit, "HealthChecker")` **propagates that + ImportError instead of returning `False`**, and so does `getattr` with a default; probe with + `importlib.util.find_spec("grpc_health")` or catch the ImportError. Tracing, metrics and the deadline budget layers are left out of the chain, with a log line, when their extra is absent — the chain still builds and the calls still run. 20. **There is no sync API and no thread safety.** Everything here assumes one event loop. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index f3f236b..ff1976b 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -73,9 +73,9 @@ No `timeout` block, no timeout interceptor — and therefore no deadline. These are **optional** and read through `getattr`, so a minimal settings object stays valid: `credentials`, `options`, `compression` (`GrpcChannelExtrasProtocol`), plus `sensitive_headers` and -`metrics_registry` (`GrpcObservabilityExtrasProtocol`). Both protocols live in -`grpc_client_kit.protocols`, along with `FullGrpcClientSettingsProtocol` for -settings that carry everything. +`metrics_registry` (`GrpcObservabilityExtrasProtocol`). Both protocols are +exported from `grpc_client_kit` like the rest, along with +`FullGrpcClientSettingsProtocol` for settings that carry everything. `connectivity` is read the same way — a `ConnectivityConfig` under that name [tunes the channels](channels.md#from-a-settings-object), its absence leaves diff --git a/docs/guide/deadlines.md b/docs/guide/deadlines.md index 82933ac..546070a 100644 --- a/docs/guide/deadlines.md +++ b/docs/guide/deadlines.md @@ -78,10 +78,14 @@ grants. Two things worth knowing before you wire it: -- **There is no settings block for this layer.** A - [factory-built chain](configuration.md#settings-objects) carries timeouts, - retries and the breaker; a chain that propagates budgets is a hand-built one, - handed to `GrpcClient(interceptors=...)`. Not to +- **A settings object can carry this layer too.** A `deadline_budget` block + with `reserve_for_next` is read from + [the settings object](configuration.md#settings-objects) with `getattr` — so + a factory-built chain gets the layer in + [its proper place](#where-it-sits-in-the-chain), with no hand-built chain + involved. What settings cannot express is anything finer — per-method caps + above all; that is what `build_interceptors` is for, and the chain it returns + goes to `GrpcClient(interceptors=...)`. Not to `create_client(interceptors=...)`, which places what it is given in the outer slot, above the layers this one has to read — [why](configuration.md#a-settings-object-end-to-end). diff --git a/docs/guide/health.md b/docs/guide/health.md index 1944bb4..0bf5258 100644 --- a/docs/guide/health.md +++ b/docs/guide/health.md @@ -20,14 +20,24 @@ pool = ChannelPool(health_checker=checker) # pooled channels get health flags t await checker.stop() # closes the probe channels ``` -Each probe asks for the **overall** server status — the empty service name of -the Health v1 protocol — rather than a per-service one, so a target is healthy -when the process behind it says it is serving at all. +Each probe asks for the service named by `service`, which defaults to the +empty name of the Health v1 protocol — the **overall** server status, so a +target is healthy when the process behind it says it is serving at all. Pass +`service="users.v1.Users"` to ask about one service instead, which is what the +standard protocol is for when a process serves several. `HealthChecker` is resolved lazily, so `import grpc_client_kit` works on a bare install; touching `grpc_client_kit.HealthChecker` without the extra -raises an `ImportError` naming it. `HealthCheckerNotRunningError` lives in -`grpc_client_kit.health`, which likewise needs the extra. +raises an `ImportError` naming it. It is an `ImportError` rather than an +`AttributeError` on purpose — a missing extra is an install problem and should +say so — which does mean `hasattr(grpc_client_kit, "HealthChecker")` raises it +rather than answering `False`, and `getattr` with a default does not swallow it +either. Probe for the extra with `importlib.util.find_spec("grpc_health")`, or +catch the `ImportError`. + +`HealthCheckerNotRunningError` is a top-level export from `grpc_client_kit` +and needs no extra at all: the caller who meets it is a caller of a balancer, +and a balancer works on a bare install. ## An unchecked target is not a healthy target @@ -79,7 +89,7 @@ Three reasons, each sufficient on its own: - a pooled channel carries the client's interceptor chain, so probing through it would pollute client metrics and count towards circuit breakers; -- pooled channels belong to application traffic and its lifecycle, while +- pooled channels belong to application traffic and its lifecycle, while monitoring must run on its own schedule; - a failed probe closes its channel to force a reconnect, which must never happen to a channel that application RPCs are holding. @@ -90,14 +100,19 @@ Three reasons, each sufficient on its own: | :--- | :--- | :--- | | `check_interval` | `30.0` | Seconds between checks of a healthy target | | `timeout` | `5.0` | Budget for one probe RPC | -| `max_backoff` | `300.0` | Cap on the backoff of a failing target | +| `service` | `""` | Service to probe; the empty name asks about the server as a whole | +| `max_backoff` | `300.0` | Cap on the backoff of a failing target; the checker's own default, not read from settings | The factory builds a checker only when the settings carry both a -`health_checker` block **and** `targets`, and passes it `check_interval`, -`timeout`, `insecure` and `credentials`. It does **not** forward `options` or -`compression`: if your application channels need specific channel options for -the probes to negotiate HTTP/2 identically, construct `HealthChecker` -yourself and hand it to the pool and the balancer. +`health_checker` block **and** `targets`. It passes `check_interval` and +`timeout` from that block, the block's optional `service`, and `insecure`, +`credentials`, `options` and `compression` from the settings themselves — the +last two so the probes negotiate HTTP/2 exactly as the application channels do, +rather than through a differently configured connection. + +Anything the settings cannot express — `on_status_change`, `max_backoff`, +`fail_fast_callback` — means constructing `HealthChecker` yourself and handing +it to the pool and the balancer. Entering the factory's `async with` starts the checker; leaving it stops the checker and closes its probe channels. A factory used without that block warns diff --git a/docs/guide/resilience.md b/docs/guide/resilience.md index 268c54b..e356a76 100644 --- a/docs/guide/resilience.md +++ b/docs/guide/resilience.md @@ -1,289 +1,291 @@ -# Resilience - -Five layers decide how long a call may take, whether it may wait for a -connection, how often it may be repeated, and when it should not be attempted -at all. They nest in that order — timeout, deadline budget, wait-for-ready, -retry, circuit breaker — and that nesting is the whole design; see -[the chain](interceptors.md#the-chain). - -Propagating the caller's remaining time is a topic of its own, since half of it -is the caller's job rather than the kit's: -[Deadline budgets](deadlines.md) covers it. - -## Timeouts - -**A timeout is the budget of an entire call, retries included.** -`AsyncTimeoutInterceptor` runs once per call; the retry layer nested below it -divides what that budget leaves. Coming from a per-attempt model, this is the -one thing to unlearn: `max_attempts × timeout` is not how long a call can -take. - -```python -from grpc_client_kit import TimeoutConfig - -TimeoutConfig( - default=10.0, - per_method={ - "/orders.v1.Orders/Export": 120.0, - "/orders.v1.Orders/Stream": None, # explicitly unbounded - }, -) -``` - -- A deadline the caller already set wins when it is **smaller**: a per-call - deadline may tighten the configured budget, never loosen it. -- `None` or `0` means "no deadline". A per-method `None` disables the budget - for that method only, overriding `default`. -- `TimeoutConfig(default=None)` with no `per_method` entries adds **no - interceptor at all** — a pass-through layer would still cost a hop per call - and, since the chain is part of the pool key, a separate channel. - -A settings `timeout` block only carries `default`. Per-method budgets need a -hand-built chain. - -Two layers may narrow the deadline further before the call is issued: the -[request budget](deadlines.md), which trims it to what the caller's request has -left, and gRPC itself, which never lets a call outlive the deadline its own -details carry. Nothing ever widens it. - -## Waiting for a connection - -A `grpc.aio` channel connects lazily, so a call made before the connection is -up fails immediately with `UNAVAILABLE`. That is the burst of errors every pod -produces in the first second of its life, and every client produces again after -a backend restart: failures that describe the *channel's age* rather than the -service's health. gRPC's answer is the `wait_for_ready` flag, and this layer is -where it gets configured — globally or per method, the way deadlines are. - -```python -from grpc_client_kit import WaitForReadyConfig - -WaitForReadyConfig( - default=True, - per_method={ - "/orders.v1.Orders/Probe": False, # this one must fail fast - "/orders.v1.Orders/Export": None, # exempt: left exactly as it arrives - }, - require_deadline=True, -) -``` - -- A method resolving to `True` is issued with `wait_for_ready=True`; `False` - is issued fail-fast explicitly; `None` — globally or per method — leaves the - call untouched. -- **A value the caller set at the call site always wins.** That is the only way - to opt one call out of a policy set for the whole client. - -### Without a deadline, this is how you hang a call forever - -On its own the flag does not remove a failure mode, it swaps one for another: -the call no longer fails fast, it waits — and a wait for a backend that never -comes back never ends by itself. `UNAVAILABLE` in 200 ms is a bad answer; -nothing at all, for the rest of the process's life, is a worse one. - -Bounded by a deadline, the same trade is one-sided: the call either connects -and runs, or ends in `DEADLINE_EXCEEDED` after exactly the time it was allowed -— which is what the caller asked for either way. Hence `require_deadline=True`, -the default: **waiting is enabled only for calls that carry a deadline.** A -call without one is left fail-fast and reported once per interceptor instance, -at `WARNING`, naming the method and the two ways out (configure a timeout for -it, or accept the risk with `require_deadline=False`). - -A chain built by this kit carries a deadline by default, so the interlock -rarely bites — but a `TimeoutConfig(default=None)` with no per-method entry, -or no `timeout` block at all, is exactly the configuration in which it does. - -### What it changes downstream - -Turning waiting on quietly rewrites which status a broken backend produces, and -two layers below read that status: - -- **Connection failures stop being retryable.** `UNAVAILABLE` is in - [`DEFAULT_RETRYABLE_CODES`](#retries); `DEADLINE_EXCEEDED` deliberately is - not. A call that used to burn three attempts on a backend that was down now - spends its deadline waiting for that backend to come back instead — usually - the better bargain, and never the same one. -- **The circuit breaker is unaffected.** Its failure set is wider than the - retryable set and contains both codes, so a backend that stays away still - trips its circuit; see - [sizing the breaker](#sizing-the-breaker-against-the-retries). -- **A dead backend now costs a full deadline instead of milliseconds.** That is - the price of the trade, paid by every call while the backend is away, and the - reason the deadline bounding it should be one you would actually be willing - to wait. - -There is no settings block for this layer either: like -[deadline budgets](deadlines.md#adding-the-layer), it is added to a hand-built -chain via `build_interceptors(wait_for_ready=...)`. It sits below the layers -that settle the deadline — it reads that deadline to decide — and above retry, -where one pass suffices, since each attempt is rebuilt from the details this -layer already wrote. - -`examples/deadline_propagation.py` shows both halves against one address: a -call refusing to wait reports `UNAVAILABLE` in 0.1 s, and the same call with -the flag waits for a server started 0.4 s later, paying for the wait out of its -request budget. - -## Retries - -```python -from grpc_client_kit import RetryConfig - -RetryConfig( - max_attempts=3, # total, first attempt included - initial_backoff=0.1, - max_backoff=10.0, - backoff_multiplier=2.0, - jitter=0.1, # backoff × (1 ± jitter) - retryable_codes=None, # None = DEFAULT_RETRYABLE_CODES; empty set = never retry - retry_streaming=False, - idempotent_methods=None, -) -``` - -`grpc_client_kit.interceptors.DEFAULT_RETRYABLE_CODES` is -`{UNAVAILABLE, RESOURCE_EXHAUSTED}` and nothing else. Both mean the attempt -was rejected before the server application saw the request: `UNAVAILABLE` -comes from connection failures and from servers that are draining, -`RESOURCE_EXHAUSTED` from quota and flow-control checks that run ahead of the -handler. - -### Retry safety - -Retrying an RPC the server already executed duplicates its side effects, so -what gets retried is deliberately narrow — and the default is a **compromise, -not a guarantee**. - -- **"Usually" is not "always".** Measured against a live server, both default - codes can follow a request that *was* executed: a server dying mid-handler - surfaces as `UNAVAILABLE` (the retry then re-executes the same logical - request on the restarted server), and a handler is free to abort with - `RESOURCE_EXHAUSTED` after a write. Where a duplicate write is unaffordable, - set `idempotent_methods` — with the whitelist in place, nothing outside it - is ever retried. -- **`INTERNAL` is deliberately absent** from the default set. It is raised by - the handler itself, so the write has very likely been applied — retrying - duplicates it with certainty rather than in the corner cases. The same - reasoning excludes `UNKNOWN`, `ABORTED` and `DEADLINE_EXCEEDED`. -- **`idempotent_methods` is a whitelist for every call kind.** When it is set, - a method outside it is not retried even on a retryable code — the tool for - widening `retryable_codes` per method rather than across your whole API. -- **Streaming responses need that whitelist.** Restarting a unary-stream call - replays items the consumer has already seen, so `retry_streaming=True` alone - is not enough: the method must also appear in `idempotent_methods`, and each - restart is logged as a warning. -- **Streaming requests are never retried.** The request iterator is consumed - by the first attempt and cannot be replayed without buffering it whole. -- **A tripped circuit is not retried.** `CircuitBreakerOpenError` carries - `UNAVAILABLE`, which is retryable by default; the retry layer recognizes the - type and re-raises it at once, instead of hammering a breaker that exists to - stop exactly that. -- **Never stack kit retries on a native `retryPolicy`.** A service config's - retries run inside the channel, below every interceptor, so the two layers - multiply — 3 × 3 = 9 requests reach the server, invisibly to the kit's logs - and metrics. The client warns when it sees both configured; see - [Native gRPC or the kit?](native-vs-kit.md). - -### Retries inside the call budget - -The relative timeout on the call details is the budget for the whole call, so -the retry layer converts it into a monotonic deadline **once**, on entry. -Before each attempt it recomputes what is left and re-expresses it as the -relative timeout gRPC understands; before each backoff it checks whether the -wait alone would outlive the budget, and abandons the retry if it would, -propagating the original error. Without this, three attempts of a ten-second -call would stretch it to thirty. - -Backoff is `initial_backoff × backoff_multiplier^(attempt-1)`, multiplied by -`(1 ± jitter)` and capped at `max_backoff`. - -## Circuit breaker - -```python -from grpc_client_kit import CircuitBreakerConfig - -CircuitBreakerConfig( - fail_threshold=5, # consecutive failures CLOSED → OPEN - recovery_timeout=60.0, # seconds OPEN before a trial call is allowed - half_open_max_calls=1, # concurrent trial calls in HALF-OPEN - max_methods=1000, # LRU bound on tracked methods - metrics=my_registry, # optional state gauge -) -``` - -States are the usual three. CLOSED counts consecutive failures and resets the -count on any success. At `fail_threshold` the circuit goes OPEN and calls fail -immediately with `CircuitBreakerOpenError` — no network involved. After -`recovery_timeout` the next call moves it to HALF-OPEN, where at most -`half_open_max_calls` trial calls run concurrently: one success closes the -circuit, one failure reopens it. A streaming call holds its trial slot until -the stream ends, which is also when its verdict is recorded. - -Only failures that say something about the server's health count: -`UNAVAILABLE`, `DEADLINE_EXCEEDED`, `INTERNAL`, `RESOURCE_EXHAUSTED`, -`ABORTED`, `UNKNOWN`, `DATA_LOSS`. Application outcomes such as `NOT_FOUND` -never trip anything, however many of them there are. Non-gRPC exceptions always -count; a cancellation counts as nothing, since a caller walking away says -nothing about the server. - -An LRU of `max_methods` entries bounds what is tracked, so a process calling -generated method names forever cannot leak — but eviction **never discards a -protecting circuit**. An OPEN state silently evicted would re-close the -breaker: the next call to the "protected" method would go out to a backend the -breaker had declared down, and it would take another full threshold of real -failures to open it again. The victim is therefore always a clean CLOSED -state; when every state is protecting something, the map grows past the limit -instead, with a warning — memory yields to correctness. - -Live snapshots are available at every level: the interceptor's -`get_states()`, `GrpcClient.circuit_breaker_states()` per target, and -`GrpcClientFactory.circuit_breaker_states()` across everything the factory -built. During an incident that is the first question — "is the breaker open, -or is the backend down?" — and the [metrics](observability.md#metrics) keep -the two apart as well: a local rejection is `status="rejected"`, never -`"error"`. - -### Circuit breaker isolation - -The breaker keeps state per method, in the interceptor instance. Since an -instance belongs to exactly one channel, and a channel to exactly one target, -giving each target its own instance makes the state effectively per -`(target, method)` — so one failing member of a load-balanced set cannot trip -the breaker for its healthy peers. `GrpcClientFactory` does this by handing -`GrpcClient` an `interceptor_factory`; a shared `interceptors` list gives up -the isolation on purpose. - -### Sizing the breaker against the retries - -The breaker is the innermost layer, so it counts **attempts, not calls** — -and the retry layer above it is what manufactures those attempts. One logical -call therefore contributes up to `max_attempts` consecutive failures to its -counter. - -That makes `fail_threshold ≤ max_attempts` a configuration that defeats -itself: - -```python -RetryConfig(max_attempts=3) -CircuitBreakerConfig(fail_threshold=2) # a single call can open its own circuit -``` - -Attempt 1 fails with `UNAVAILABLE` and is counted. The retry layer issues -attempt 2, which fails and reaches the threshold — the circuit opens. The -retry layer then tries attempt 3, and the breaker it just tripped refuses it -with `CircuitBreakerOpenError`. Since a tripped circuit is never retried, that -error is what propagates: the caller is told -`UNAVAILABLE: Circuit breaker for /pkg.Service/Method is open` instead of the -status the server actually returned, the last attempt never reaches the wire, -and every following call fails fast for `recovery_timeout` seconds on the -evidence of one unlucky request. - -Keep `fail_threshold` above `max_attempts`. With the defaults (3 attempts, -threshold 5) no single call can open the circuit, and it takes roughly -`fail_threshold / max_attempts` consecutively failing calls to do so — the -number to reason about when tuning either value. - -The inverse pairing is harmless but worth knowing: the breaker's failure set -is wider than the retryable set, so an attempt can count against the circuit -without ever being retried — a method that keeps hitting its deadline opens -its circuit with no retry issued at all. +# Resilience + +Five layers decide how long a call may take, whether it may wait for a +connection, how often it may be repeated, and when it should not be attempted +at all. They nest in that order — timeout, deadline budget, wait-for-ready, +retry, circuit breaker — and that nesting is the whole design; see +[the chain](interceptors.md#the-chain). + +Propagating the caller's remaining time is a topic of its own, since half of it +is the caller's job rather than the kit's: +[Deadline budgets](deadlines.md) covers it. + +## Timeouts + +**A timeout is the budget of an entire call, retries included.** +`AsyncTimeoutInterceptor` runs once per call; the retry layer nested below it +divides what that budget leaves. Coming from a per-attempt model, this is the +one thing to unlearn: `max_attempts × timeout` is not how long a call can +take. + +```python +from grpc_client_kit import TimeoutConfig + +TimeoutConfig( + default=10.0, + per_method={ + "/orders.v1.Orders/Export": 120.0, + "/orders.v1.Orders/Stream": None, # explicitly unbounded + }, +) +``` + +- A deadline the caller already set wins when it is **smaller**: a per-call + deadline may tighten the configured budget, never loosen it. +- `None` or `0` means "no deadline". A per-method `None` disables the budget + for that method only, overriding `default`. +- `TimeoutConfig(default=None)` with no `per_method` entries adds **no + interceptor at all** — a pass-through layer would still cost a hop per call + and, since the chain is part of the pool key, a separate channel. + +A settings `timeout` block only carries `default`. Per-method budgets need a +hand-built chain. + +Two layers may narrow the deadline further before the call is issued: the +[request budget](deadlines.md), which trims it to what the caller's request has +left, and gRPC itself, which never lets a call outlive the deadline its own +details carry. Nothing ever widens it. + +## Waiting for a connection + +A `grpc.aio` channel connects lazily, so a call made before the connection is +up fails immediately with `UNAVAILABLE`. That is the burst of errors every pod +produces in the first second of its life, and every client produces again after +a backend restart: failures that describe the *channel's age* rather than the +service's health. gRPC's answer is the `wait_for_ready` flag, and this layer is +where it gets configured — globally or per method, the way deadlines are. + +```python +from grpc_client_kit import WaitForReadyConfig + +WaitForReadyConfig( + default=True, + per_method={ + "/orders.v1.Orders/Probe": False, # this one must fail fast + "/orders.v1.Orders/Export": None, # exempt: left exactly as it arrives + }, + require_deadline=True, +) +``` + +- A method resolving to `True` is issued with `wait_for_ready=True`; `False` + is issued fail-fast explicitly; `None` — globally or per method — leaves the + call untouched. +- **A value the caller set at the call site always wins.** That is the only way + to opt one call out of a policy set for the whole client. + +### Without a deadline, this is how you hang a call forever + +On its own the flag does not remove a failure mode, it swaps one for another: +the call no longer fails fast, it waits — and a wait for a backend that never +comes back never ends by itself. `UNAVAILABLE` in 200 ms is a bad answer; +nothing at all, for the rest of the process's life, is a worse one. + +Bounded by a deadline, the same trade is one-sided: the call either connects +and runs, or ends in `DEADLINE_EXCEEDED` after exactly the time it was allowed +— which is what the caller asked for either way. Hence `require_deadline=True`, +the default: **waiting is enabled only for calls that carry a deadline.** A +call without one is left fail-fast and reported once per interceptor instance, +at `WARNING`, naming the method and the two ways out (configure a timeout for +it, or accept the risk with `require_deadline=False`). + +A chain built by this kit carries a deadline by default, so the interlock +rarely bites — but a `TimeoutConfig(default=None)` with no per-method entry, +or no `timeout` block at all, is exactly the configuration in which it does. + +### What it changes downstream + +Turning waiting on quietly rewrites which status a broken backend produces, and +two layers below read that status: + +- **Connection failures stop being retryable.** `UNAVAILABLE` is in + [`DEFAULT_RETRYABLE_CODES`](#retries); `DEADLINE_EXCEEDED` deliberately is + not. A call that used to burn three attempts on a backend that was down now + spends its deadline waiting for that backend to come back instead — usually + the better bargain, and never the same one. +- **The circuit breaker is unaffected.** Its failure set is wider than the + retryable set and contains both codes, so a backend that stays away still + trips its circuit; see + [sizing the breaker](#sizing-the-breaker-against-the-retries). +- **A dead backend now costs a full deadline instead of milliseconds.** That is + the price of the trade, paid by every call while the backend is away, and the + reason the deadline bounding it should be one you would actually be willing + to wait. + +A settings object reaches this layer the same way it reaches +[deadline budgets](deadlines.md#adding-the-layer): an optional `wait_for_ready` +block with `default`, `per_method` and `require_deadline`, read with `getattr` +and turned into the interceptor by the factory. A hand-built chain asks for it +as `build_interceptors(wait_for_ready=...)`. Either way it sits below the layers +that settle the deadline — it reads that deadline to decide — and above retry, +where one pass suffices, since each attempt is rebuilt from the details this +layer already wrote. + +`examples/deadline_propagation.py` shows both halves against one address: a +call refusing to wait reports `UNAVAILABLE` in 0.1 s, and the same call with +the flag waits for a server started 0.4 s later, paying for the wait out of its +request budget. + +## Retries + +```python +from grpc_client_kit import RetryConfig + +RetryConfig( + max_attempts=3, # total, first attempt included + initial_backoff=0.1, + max_backoff=10.0, + backoff_multiplier=2.0, + jitter=0.1, # backoff × (1 ± jitter) + retryable_codes=None, # None = DEFAULT_RETRYABLE_CODES; empty set = never retry + retry_streaming=False, + idempotent_methods=None, +) +``` + +`grpc_client_kit.interceptors.DEFAULT_RETRYABLE_CODES` is +`{UNAVAILABLE, RESOURCE_EXHAUSTED}` and nothing else. Both mean the attempt +was rejected before the server application saw the request: `UNAVAILABLE` +comes from connection failures and from servers that are draining, +`RESOURCE_EXHAUSTED` from quota and flow-control checks that run ahead of the +handler. + +### Retry safety + +Retrying an RPC the server already executed duplicates its side effects, so +what gets retried is deliberately narrow — and the default is a **compromise, +not a guarantee**. + +- **"Usually" is not "always".** Measured against a live server, both default + codes can follow a request that *was* executed: a server dying mid-handler + surfaces as `UNAVAILABLE` (the retry then re-executes the same logical + request on the restarted server), and a handler is free to abort with + `RESOURCE_EXHAUSTED` after a write. Where a duplicate write is unaffordable, + set `idempotent_methods` — with the whitelist in place, nothing outside it + is ever retried. +- **`INTERNAL` is deliberately absent** from the default set. It is raised by + the handler itself, so the write has very likely been applied — retrying + duplicates it with certainty rather than in the corner cases. The same + reasoning excludes `UNKNOWN`, `ABORTED` and `DEADLINE_EXCEEDED`. +- **`idempotent_methods` is a whitelist for every call kind.** When it is set, + a method outside it is not retried even on a retryable code — the tool for + widening `retryable_codes` per method rather than across your whole API. +- **Streaming responses need that whitelist.** Restarting a unary-stream call + replays items the consumer has already seen, so `retry_streaming=True` alone + is not enough: the method must also appear in `idempotent_methods`, and each + restart is logged as a warning. +- **Streaming requests are never retried.** The request iterator is consumed + by the first attempt and cannot be replayed without buffering it whole. +- **A tripped circuit is not retried.** `CircuitBreakerOpenError` carries + `UNAVAILABLE`, which is retryable by default; the retry layer recognizes the + type and re-raises it at once, instead of hammering a breaker that exists to + stop exactly that. +- **Never stack kit retries on a native `retryPolicy`.** A service config's + retries run inside the channel, below every interceptor, so the two layers + multiply — 3 × 3 = 9 requests reach the server, invisibly to the kit's logs + and metrics. The client warns when it sees both configured; see + [Native gRPC or the kit?](native-vs-kit.md). + +### Retries inside the call budget + +The relative timeout on the call details is the budget for the whole call, so +the retry layer converts it into a monotonic deadline **once**, on entry. +Before each attempt it recomputes what is left and re-expresses it as the +relative timeout gRPC understands; before each backoff it checks whether the +wait alone would outlive the budget, and abandons the retry if it would, +propagating the original error. Without this, three attempts of a ten-second +call would stretch it to thirty. + +Backoff is `initial_backoff × backoff_multiplier^(attempt-1)`, multiplied by +`(1 ± jitter)` and capped at `max_backoff`. + +## Circuit breaker + +```python +from grpc_client_kit import CircuitBreakerConfig + +CircuitBreakerConfig( + fail_threshold=5, # consecutive failures CLOSED → OPEN + recovery_timeout=60.0, # seconds OPEN before a trial call is allowed + half_open_max_calls=1, # concurrent trial calls in HALF-OPEN + max_methods=1000, # LRU bound on tracked methods + metrics=my_registry, # optional state gauge +) +``` + +States are the usual three. CLOSED counts consecutive failures and resets the +count on any success. At `fail_threshold` the circuit goes OPEN and calls fail +immediately with `CircuitBreakerOpenError` — no network involved. After +`recovery_timeout` the next call moves it to HALF-OPEN, where at most +`half_open_max_calls` trial calls run concurrently: one success closes the +circuit, one failure reopens it. A streaming call holds its trial slot until +the stream ends, which is also when its verdict is recorded. + +Only failures that say something about the server's health count: +`UNAVAILABLE`, `DEADLINE_EXCEEDED`, `INTERNAL`, `RESOURCE_EXHAUSTED`, +`ABORTED`, `UNKNOWN`, `DATA_LOSS`. Application outcomes such as `NOT_FOUND` +never trip anything, however many of them there are. Non-gRPC exceptions always +count; a cancellation counts as nothing, since a caller walking away says +nothing about the server. + +An LRU of `max_methods` entries bounds what is tracked, so a process calling +generated method names forever cannot leak — but eviction **never discards a +protecting circuit**. An OPEN state silently evicted would re-close the +breaker: the next call to the "protected" method would go out to a backend the +breaker had declared down, and it would take another full threshold of real +failures to open it again. The victim is therefore always a clean CLOSED +state; when every state is protecting something, the map grows past the limit +instead, with a warning — memory yields to correctness. + +Live snapshots are available at every level: the interceptor's +`get_states()`, `GrpcClient.circuit_breaker_states()` per target, and +`GrpcClientFactory.circuit_breaker_states()` across everything the factory +built. During an incident that is the first question — "is the breaker open, +or is the backend down?" — and the [metrics](observability.md#metrics) keep +the two apart as well: a local rejection is `status="rejected"`, never +`"error"`. + +### Circuit breaker isolation + +The breaker keeps state per method, in the interceptor instance. Since an +instance belongs to exactly one channel, and a channel to exactly one target, +giving each target its own instance makes the state effectively per +`(target, method)` — so one failing member of a load-balanced set cannot trip +the breaker for its healthy peers. `GrpcClientFactory` does this by handing +`GrpcClient` an `interceptor_factory`; a shared `interceptors` list gives up +the isolation on purpose. + +### Sizing the breaker against the retries + +The breaker is the innermost layer, so it counts **attempts, not calls** — +and the retry layer above it is what manufactures those attempts. One logical +call therefore contributes up to `max_attempts` consecutive failures to its +counter. + +That makes `fail_threshold ≤ max_attempts` a configuration that defeats +itself: + +```python +RetryConfig(max_attempts=3) +CircuitBreakerConfig(fail_threshold=2) # a single call can open its own circuit +``` + +Attempt 1 fails with `UNAVAILABLE` and is counted. The retry layer issues +attempt 2, which fails and reaches the threshold — the circuit opens. The +retry layer then tries attempt 3, and the breaker it just tripped refuses it +with `CircuitBreakerOpenError`. Since a tripped circuit is never retried, that +error is what propagates: the caller is told +`UNAVAILABLE: Circuit breaker for /pkg.Service/Method is open` instead of the +status the server actually returned, the last attempt never reaches the wire, +and every following call fails fast for `recovery_timeout` seconds on the +evidence of one unlucky request. + +Keep `fail_threshold` above `max_attempts`. With the defaults (3 attempts, +threshold 5) no single call can open the circuit, and it takes roughly +`fail_threshold / max_attempts` consecutively failing calls to do so — the +number to reason about when tuning either value. + +The inverse pairing is harmless but worth knowing: the breaker's failure set +is wider than the retryable set, so an attempt can count against the circuit +without ever being retried — a method that keeps hitting its deadline opens +its circuit with no retry issued at all. diff --git a/grpc_client_kit/__init__.py b/grpc_client_kit/__init__.py index ac14a18..3b464e9 100644 --- a/grpc_client_kit/__init__.py +++ b/grpc_client_kit/__init__.py @@ -59,8 +59,11 @@ ChannelProviderProtocol, CircuitBreakerMetricsProtocol, CircuitBreakerSettingsProtocol, + FullGrpcClientSettingsProtocol, + GrpcChannelExtrasProtocol, GrpcClientMetricsProtocol, GrpcClientSettingsProtocol, + GrpcObservabilityExtrasProtocol, HealthCheckerProtocol, HealthCheckerSettingsProtocol, HealthStatusCallbackProtocol, @@ -78,8 +81,10 @@ # The public surface, deliberately curated. The extension seam (AsyncAroundClientInterceptor, # ClientCall, flatten_interceptors) is first-class: it is what custom interceptors are written -# against. Pool internals (ChannelWrapper, chain_token) are deliberately NOT here — exporting them -# would freeze the pool's implementation into the compatibility contract. +# against. Every protocol in `protocols.__all__` is re-exported here, so a settings object can be +# written against the package alone. Internals — the pool's ChannelWrapper and chain_token, the +# circuit breaker's MethodCircuitState — are deliberately NOT here: exporting them would freeze +# those implementations into the compatibility contract. __all__ = [ "AsyncAroundClientInterceptor", "AsyncCircuitBreakerInterceptor", @@ -104,12 +109,15 @@ "DeadlineBudgetConfig", "DeadlineBudgetExhaustedError", "DeadlineBudgetProtocol", + "FullGrpcClientSettingsProtocol", + "GrpcChannelExtrasProtocol", "GrpcClient", "GrpcClientConfig", "GrpcClientFactory", "GrpcClientKitError", "GrpcClientMetricsProtocol", "GrpcClientSettingsProtocol", + "GrpcObservabilityExtrasProtocol", "HealthChecker", "HealthCheckerNotRunningError", "HealthCheckerProtocol", @@ -148,6 +156,12 @@ def __getattr__(name: str) -> Any: ``HealthChecker`` needs the [health] extra, so importing this package must not import it: a module-level import would make ``import grpc_client_kit`` fail on a bare install. + A missing extra raises ``ImportError``, not ``AttributeError``, so that a broken install + says so instead of looking like a name that never existed. The cost is that ``hasattr`` and + ``getattr`` with a default do not swallow it — they propagate the ImportError — so probe for + the extra with ``importlib.util.find_spec("grpc_health")`` or catch the ImportError, not + with ``hasattr(grpc_client_kit, "HealthChecker")``. + Args: name: The attribute being looked up. @@ -156,7 +170,8 @@ def __getattr__(name: str) -> Any: Raises: AttributeError: If the package has no such attribute. - ImportError: If the attribute needs an extra that is not installed. + ImportError: If the attribute needs an extra that is not installed. Deliberately not an + AttributeError: see above. """ if name == "HealthChecker": from .factory import _load_health_checker # noqa: PLC0415 - lazy: needs the [health] extra diff --git a/grpc_client_kit/interceptors/circuit_breaker.py b/grpc_client_kit/interceptors/circuit_breaker.py index 9594a2b..cf05dca 100644 --- a/grpc_client_kit/interceptors/circuit_breaker.py +++ b/grpc_client_kit/interceptors/circuit_breaker.py @@ -399,10 +399,12 @@ def _should_record_failure(self, code: grpc.StatusCode | None) -> bool: ) +# `MethodCircuitState` is deliberately absent: it is the breaker's mutable bookkeeping, never a +# type a caller receives — `get_states()` hands out `CircuitBreakerStatus` snapshots — and +# declaring it public would freeze this implementation into the compatibility contract. __all__ = [ "AsyncCircuitBreakerInterceptor", "CircuitBreakerOpenError", "CircuitBreakerStatus", "CircuitState", - "MethodCircuitState", ] diff --git a/grpc_client_kit/protocols.py b/grpc_client_kit/protocols.py index a5d1d00..b004f34 100644 --- a/grpc_client_kit/protocols.py +++ b/grpc_client_kit/protocols.py @@ -1,449 +1,451 @@ -"""Structural protocols (duck-typing seams) for gRPC client settings and collaborators. - -Settings fields are declared as read-only properties, not plain attributes: protocol -attributes are invariant under type checking, while read-only properties are covariant, -so an implementation may narrow a field's type (e.g. a pydantic model typing `strategy` -as a `Literal`) and still satisfy the protocol. - -Settings protocols describe only the fields real settings objects are required to carry. -Optional blocks live in their own protocols (`GrpcChannelExtrasProtocol`, -`GrpcObservabilityExtrasProtocol`) so that `isinstance` against a settings protocol stays -a meaningful check instead of failing on fields nobody defines. -""" - -from __future__ import annotations - -from typing import ( - Any, - Protocol, - runtime_checkable, -) - -import grpc -import grpc.aio - - -@runtime_checkable -class HealthCheckerProtocol(Protocol): - """Protocol for target health monitoring.""" - - async def is_healthy(self, target: str) -> bool: - """Check if target is healthy (from cache). - - Implementations report health from evidence only: a target that has never been - checked is not healthy. An implementation that cannot produce evidence at all - (its check loop is not running) may raise instead of answering. - - Args: - target: The target address (host:port) - - Returns: - True if the last check reported healthy, False otherwise - """ - ... - - async def check_health(self, target: str) -> bool: - """Perform active health check for target. - - Args: - target: The target address (host:port) - - Returns: - True if healthy, False otherwise - """ - ... - - -@runtime_checkable -class ChannelProviderProtocol(Protocol): - """Protocol for gRPC channel management.""" - - async def get_channel( - self, - target: str, - insecure: bool = False, - credentials: grpc.ChannelCredentials | None = None, - options: list[tuple[str, Any]] | None = None, - compression: grpc.Compression | None = None, - interceptors: list[grpc.aio.ClientInterceptor] | None = None, - ) -> grpc.aio.Channel: - """Get or create a gRPC channel for the target. - - Args: - target: The target address (host:port) - insecure: Whether to use insecure channel - credentials: Optional channel credentials for secure channel - options: Optional gRPC channel options - compression: Optional gRPC compression - interceptors: Optional list of interceptors - - Returns: - An async gRPC channel - """ - ... - - async def close_all(self, grace: float | None = None) -> None: - """Close all pooled channels and release resources.""" - ... - - async def update_channel_health(self, target: str, is_healthy: bool) -> None: - """Update health status for a specific target. - - Args: - target: The target address (host:port) - is_healthy: Whether the target is healthy - """ - ... - - -@runtime_checkable -class HealthStatusCallbackProtocol(Protocol): - """Protocol for health status change callbacks.""" - - async def __call__(self, target: str, is_healthy: bool) -> None: - """Called when health status of a target changes. - - Args: - target: The target address (host:port) - is_healthy: True if target is healthy, False otherwise - """ - ... - - -@runtime_checkable -class GrpcClientMetricsProtocol(Protocol): - """Protocol for gRPC client metrics collection.""" - - def record_request( - self, - service: str, - method: str, - rpc_type: str, - status: str, - grpc_code: str, - duration: float, - ) -> None: - """Record a completed gRPC request. - - Args: - service: Name of the service. - method: Name of the method. - rpc_type: Type of RPC (unary_unary, unary_stream, etc.). - status: Status of the request (success, error, cancelled). - grpc_code: gRPC status code name. - duration: Request duration in seconds. - """ - ... - - def record_inflight_delta( - self, - service: str, - method: str, - rpc_type: str, - delta: int, - ) -> None: - """Record a change in in-flight requests. - - Args: - service: Name of the service. - method: Name of the method. - rpc_type: Type of RPC. - delta: Change in in-flight requests (e.g., +1 or -1). - """ - ... - - def record_pool_stats( - self, - active_channels: int, - idle_targets: int, - ) -> None: - """Record channel pool statistics. - - Args: - active_channels: Total number of active channels in the pool. - idle_targets: Number of targets currently in the pool. - """ - ... - - -@runtime_checkable -class RetryMetricsProtocol(Protocol): - """Optional extension: retry attempts, invisible to `record_request` by design. - - The metrics layer sits above the retry layer and records one entry per *logical* call, so a - retry storm — N wire attempts collapsing into one success — cannot be seen through - `GrpcClientMetricsProtocol` alone. A registry that also implements this protocol gets told - about every retry the moment it is scheduled. - """ - - def record_retry(self, service: str, method: str, attempt: int, grpc_code: str) -> None: - """Record one scheduled retry. - - Args: - service: Name of the service. - method: Name of the method. - attempt: Number of the upcoming attempt (1 is the first retry). - grpc_code: Status code name of the failure that caused the retry. - """ - ... - - -@runtime_checkable -class CircuitBreakerMetricsProtocol(Protocol): - """Optional extension: circuit breaker state transitions and rejections. - - A rejection by an open breaker never touches the network, yet through `record_request` alone - it is indistinguishable from a backend that really failed. A registry that also implements - this protocol can chart the breaker itself: its state per method, and how many calls it - refused locally. - """ - - def record_circuit_state(self, method: str, state: str) -> None: - """Record a circuit state transition. - - Args: - method: Full gRPC method path. - state: The new state (``closed``, ``open`` or ``half-open``). - """ - ... - - def record_circuit_rejection(self, method: str) -> None: - """Record one call refused locally by an open circuit. - - Args: - method: Full gRPC method path. - """ - ... - - -@runtime_checkable -class ChannelPoolSettingsProtocol(Protocol): - """Protocol for gRPC channel pool settings.""" - - @property - def max_channels_per_target(self) -> int: - """Maximum number of channels to keep per target.""" - ... - - @property - def idle_timeout(self) -> float: - """Time in seconds after which an idle channel is closed.""" - ... - - -@runtime_checkable -class CircuitBreakerSettingsProtocol(Protocol): - """Protocol for gRPC circuit breaker settings.""" - - @property - def fail_threshold(self) -> int: - """Number of failures before opening the circuit.""" - ... - - @property - def recovery_timeout(self) -> float: - """Time in seconds to wait before attempting recovery.""" - ... - - @property - def half_open_max_calls(self) -> int: - """Maximum number of calls allowed in half-open state.""" - ... - - -@runtime_checkable -class RetrySettingsProtocol(Protocol): - """Protocol for gRPC retry settings.""" - - @property - def max_attempts(self) -> int: - """Maximum number of attempts (including the first one).""" - ... - - @property - def initial_backoff(self) -> float: - """Initial backoff time in seconds.""" - ... - - @property - def max_backoff(self) -> float: - """Maximum backoff time in seconds.""" - ... - - @property - def backoff_multiplier(self) -> float: - """Multiplier for exponential backoff.""" - ... - - -@runtime_checkable -class TimeoutSettingsProtocol(Protocol): - """Protocol for gRPC timeout settings.""" - - @property - def default(self) -> float: - """Default timeout in seconds.""" - ... - - -@runtime_checkable -class LoadBalancerSettingsProtocol(Protocol): - """Protocol for gRPC load balancer settings.""" - - @property - def strategy(self) -> str: - """Load balancing strategy (round_robin, random, weighted).""" - ... - - @property - def weights(self) -> dict[str, float] | None: - """Weights for weighted strategy (target -> weight).""" - ... - - -@runtime_checkable -class HealthCheckerSettingsProtocol(Protocol): - """Protocol for gRPC health checker settings.""" - - @property - def check_interval(self) -> float: - """Interval between health checks in seconds.""" - ... - - @property - def timeout(self) -> float: - """Timeout for each health check in seconds.""" - ... - - -@runtime_checkable -class GrpcClientSettingsProtocol(Protocol): - """Protocol for base gRPC client settings. - - Every field here is one a settings object must carry. Anything a client may or may - not configure (credentials, channel options, redaction, metrics registry) belongs to - the extras protocols below, so that a plain settings model satisfies this one. - """ - - @property - def target(self) -> str | None: - """Single target address (host:port).""" - ... - - @property - def targets(self) -> list[str] | None: - """List of target addresses for load balancing.""" - ... - - @property - def insecure(self) -> bool: - """Whether to use insecure connection.""" - ... - - @property - def tracing_enabled(self) -> bool: - """Whether tracing is enabled.""" - ... - - @property - def metrics_enabled(self) -> bool: - """Whether metrics are enabled.""" - ... - - @property - def logging_enabled(self) -> bool: - """Whether logging is enabled.""" - ... - - @property - def pool(self) -> ChannelPoolSettingsProtocol | None: - """Channel pool settings.""" - ... - - @property - def circuit_breaker(self) -> CircuitBreakerSettingsProtocol | None: - """Circuit breaker settings.""" - ... - - @property - def retry(self) -> RetrySettingsProtocol | None: - """Retry settings.""" - ... - - @property - def timeout(self) -> TimeoutSettingsProtocol | None: - """Timeout settings.""" - ... - - @property - def balancer(self) -> LoadBalancerSettingsProtocol | None: - """Load balancer settings.""" - ... - - @property - def health_checker(self) -> HealthCheckerSettingsProtocol | None: - """Health checker settings.""" - ... - - -@runtime_checkable -class GrpcChannelExtrasProtocol(Protocol): - """Protocol for optional channel-construction settings.""" - - @property - def credentials(self) -> grpc.ChannelCredentials | None: - """Channel credentials for secure connection.""" - ... - - @property - def options(self) -> list[tuple[str, Any]] | None: - """Optional gRPC channel options.""" - ... - - @property - def compression(self) -> grpc.Compression | None: - """Optional gRPC compression.""" - ... - - -@runtime_checkable -class GrpcObservabilityExtrasProtocol(Protocol): - """Protocol for optional observability settings.""" - - @property - def sensitive_headers(self) -> set[str] | None: - """Set of header names to redact during logging.""" - ... - - @property - def metrics_registry(self) -> GrpcClientMetricsProtocol | None: - """Optional metrics registry for pool and client metrics.""" - ... - - -@runtime_checkable -class FullGrpcClientSettingsProtocol( - GrpcClientSettingsProtocol, - GrpcChannelExtrasProtocol, - GrpcObservabilityExtrasProtocol, - Protocol, -): - """Protocol for settings that carry the required fields and every optional block.""" - - -__all__ = [ - "ChannelPoolSettingsProtocol", - "ChannelProviderProtocol", - "CircuitBreakerSettingsProtocol", - "FullGrpcClientSettingsProtocol", - "GrpcChannelExtrasProtocol", - "GrpcClientMetricsProtocol", - "GrpcClientSettingsProtocol", - "GrpcObservabilityExtrasProtocol", - "HealthCheckerProtocol", - "HealthCheckerSettingsProtocol", - "HealthStatusCallbackProtocol", - "LoadBalancerSettingsProtocol", - "RetrySettingsProtocol", - "TimeoutSettingsProtocol", -] +"""Structural protocols (duck-typing seams) for gRPC client settings and collaborators. + +Settings fields are declared as read-only properties, not plain attributes: protocol +attributes are invariant under type checking, while read-only properties are covariant, +so an implementation may narrow a field's type (e.g. a pydantic model typing `strategy` +as a `Literal`) and still satisfy the protocol. + +Settings protocols describe only the fields real settings objects are required to carry. +Optional blocks live in their own protocols (`GrpcChannelExtrasProtocol`, +`GrpcObservabilityExtrasProtocol`) so that `isinstance` against a settings protocol stays +a meaningful check instead of failing on fields nobody defines. +""" + +from __future__ import annotations + +from typing import ( + Any, + Protocol, + runtime_checkable, +) + +import grpc +import grpc.aio + + +@runtime_checkable +class HealthCheckerProtocol(Protocol): + """Protocol for target health monitoring.""" + + async def is_healthy(self, target: str) -> bool: + """Check if target is healthy (from cache). + + Implementations report health from evidence only: a target that has never been + checked is not healthy. An implementation that cannot produce evidence at all + (its check loop is not running) may raise instead of answering. + + Args: + target: The target address (host:port) + + Returns: + True if the last check reported healthy, False otherwise + """ + ... + + async def check_health(self, target: str) -> bool: + """Perform active health check for target. + + Args: + target: The target address (host:port) + + Returns: + True if healthy, False otherwise + """ + ... + + +@runtime_checkable +class ChannelProviderProtocol(Protocol): + """Protocol for gRPC channel management.""" + + async def get_channel( + self, + target: str, + insecure: bool = False, + credentials: grpc.ChannelCredentials | None = None, + options: list[tuple[str, Any]] | None = None, + compression: grpc.Compression | None = None, + interceptors: list[grpc.aio.ClientInterceptor] | None = None, + ) -> grpc.aio.Channel: + """Get or create a gRPC channel for the target. + + Args: + target: The target address (host:port) + insecure: Whether to use insecure channel + credentials: Optional channel credentials for secure channel + options: Optional gRPC channel options + compression: Optional gRPC compression + interceptors: Optional list of interceptors + + Returns: + An async gRPC channel + """ + ... + + async def close_all(self, grace: float | None = None) -> None: + """Close all pooled channels and release resources.""" + ... + + async def update_channel_health(self, target: str, is_healthy: bool) -> None: + """Update health status for a specific target. + + Args: + target: The target address (host:port) + is_healthy: Whether the target is healthy + """ + ... + + +@runtime_checkable +class HealthStatusCallbackProtocol(Protocol): + """Protocol for health status change callbacks.""" + + async def __call__(self, target: str, is_healthy: bool) -> None: + """Called when health status of a target changes. + + Args: + target: The target address (host:port) + is_healthy: True if target is healthy, False otherwise + """ + ... + + +@runtime_checkable +class GrpcClientMetricsProtocol(Protocol): + """Protocol for gRPC client metrics collection.""" + + def record_request( + self, + service: str, + method: str, + rpc_type: str, + status: str, + grpc_code: str, + duration: float, + ) -> None: + """Record a completed gRPC request. + + Args: + service: Name of the service. + method: Name of the method. + rpc_type: Type of RPC (unary_unary, unary_stream, etc.). + status: Status of the request (success, error, cancelled). + grpc_code: gRPC status code name. + duration: Request duration in seconds. + """ + ... + + def record_inflight_delta( + self, + service: str, + method: str, + rpc_type: str, + delta: int, + ) -> None: + """Record a change in in-flight requests. + + Args: + service: Name of the service. + method: Name of the method. + rpc_type: Type of RPC. + delta: Change in in-flight requests (e.g., +1 or -1). + """ + ... + + def record_pool_stats( + self, + active_channels: int, + idle_targets: int, + ) -> None: + """Record channel pool statistics. + + Args: + active_channels: Total number of active channels in the pool. + idle_targets: Number of targets currently in the pool. + """ + ... + + +@runtime_checkable +class RetryMetricsProtocol(Protocol): + """Optional extension: retry attempts, invisible to `record_request` by design. + + The metrics layer sits above the retry layer and records one entry per *logical* call, so a + retry storm — N wire attempts collapsing into one success — cannot be seen through + `GrpcClientMetricsProtocol` alone. A registry that also implements this protocol gets told + about every retry the moment it is scheduled. + """ + + def record_retry(self, service: str, method: str, attempt: int, grpc_code: str) -> None: + """Record one scheduled retry. + + Args: + service: Name of the service. + method: Name of the method. + attempt: Number of the upcoming attempt (1 is the first retry). + grpc_code: Status code name of the failure that caused the retry. + """ + ... + + +@runtime_checkable +class CircuitBreakerMetricsProtocol(Protocol): + """Optional extension: circuit breaker state transitions and rejections. + + A rejection by an open breaker never touches the network, yet through `record_request` alone + it is indistinguishable from a backend that really failed. A registry that also implements + this protocol can chart the breaker itself: its state per method, and how many calls it + refused locally. + """ + + def record_circuit_state(self, method: str, state: str) -> None: + """Record a circuit state transition. + + Args: + method: Full gRPC method path. + state: The new state (``closed``, ``open`` or ``half-open``). + """ + ... + + def record_circuit_rejection(self, method: str) -> None: + """Record one call refused locally by an open circuit. + + Args: + method: Full gRPC method path. + """ + ... + + +@runtime_checkable +class ChannelPoolSettingsProtocol(Protocol): + """Protocol for gRPC channel pool settings.""" + + @property + def max_channels_per_target(self) -> int: + """Maximum number of channels to keep per target.""" + ... + + @property + def idle_timeout(self) -> float: + """Time in seconds after which an idle channel is closed.""" + ... + + +@runtime_checkable +class CircuitBreakerSettingsProtocol(Protocol): + """Protocol for gRPC circuit breaker settings.""" + + @property + def fail_threshold(self) -> int: + """Number of failures before opening the circuit.""" + ... + + @property + def recovery_timeout(self) -> float: + """Time in seconds to wait before attempting recovery.""" + ... + + @property + def half_open_max_calls(self) -> int: + """Maximum number of calls allowed in half-open state.""" + ... + + +@runtime_checkable +class RetrySettingsProtocol(Protocol): + """Protocol for gRPC retry settings.""" + + @property + def max_attempts(self) -> int: + """Maximum number of attempts (including the first one).""" + ... + + @property + def initial_backoff(self) -> float: + """Initial backoff time in seconds.""" + ... + + @property + def max_backoff(self) -> float: + """Maximum backoff time in seconds.""" + ... + + @property + def backoff_multiplier(self) -> float: + """Multiplier for exponential backoff.""" + ... + + +@runtime_checkable +class TimeoutSettingsProtocol(Protocol): + """Protocol for gRPC timeout settings.""" + + @property + def default(self) -> float: + """Default timeout in seconds.""" + ... + + +@runtime_checkable +class LoadBalancerSettingsProtocol(Protocol): + """Protocol for gRPC load balancer settings.""" + + @property + def strategy(self) -> str: + """Load balancing strategy (round_robin, random, weighted).""" + ... + + @property + def weights(self) -> dict[str, float] | None: + """Weights for weighted strategy (target -> weight).""" + ... + + +@runtime_checkable +class HealthCheckerSettingsProtocol(Protocol): + """Protocol for gRPC health checker settings.""" + + @property + def check_interval(self) -> float: + """Interval between health checks in seconds.""" + ... + + @property + def timeout(self) -> float: + """Timeout for each health check in seconds.""" + ... + + +@runtime_checkable +class GrpcClientSettingsProtocol(Protocol): + """Protocol for base gRPC client settings. + + Every field here is one a settings object must carry. Anything a client may or may + not configure (credentials, channel options, redaction, metrics registry) belongs to + the extras protocols below, so that a plain settings model satisfies this one. + """ + + @property + def target(self) -> str | None: + """Single target address (host:port).""" + ... + + @property + def targets(self) -> list[str] | None: + """List of target addresses for load balancing.""" + ... + + @property + def insecure(self) -> bool: + """Whether to use insecure connection.""" + ... + + @property + def tracing_enabled(self) -> bool: + """Whether tracing is enabled.""" + ... + + @property + def metrics_enabled(self) -> bool: + """Whether metrics are enabled.""" + ... + + @property + def logging_enabled(self) -> bool: + """Whether logging is enabled.""" + ... + + @property + def pool(self) -> ChannelPoolSettingsProtocol | None: + """Channel pool settings.""" + ... + + @property + def circuit_breaker(self) -> CircuitBreakerSettingsProtocol | None: + """Circuit breaker settings.""" + ... + + @property + def retry(self) -> RetrySettingsProtocol | None: + """Retry settings.""" + ... + + @property + def timeout(self) -> TimeoutSettingsProtocol | None: + """Timeout settings.""" + ... + + @property + def balancer(self) -> LoadBalancerSettingsProtocol | None: + """Load balancer settings.""" + ... + + @property + def health_checker(self) -> HealthCheckerSettingsProtocol | None: + """Health checker settings.""" + ... + + +@runtime_checkable +class GrpcChannelExtrasProtocol(Protocol): + """Protocol for optional channel-construction settings.""" + + @property + def credentials(self) -> grpc.ChannelCredentials | None: + """Channel credentials for secure connection.""" + ... + + @property + def options(self) -> list[tuple[str, Any]] | None: + """Optional gRPC channel options.""" + ... + + @property + def compression(self) -> grpc.Compression | None: + """Optional gRPC compression.""" + ... + + +@runtime_checkable +class GrpcObservabilityExtrasProtocol(Protocol): + """Protocol for optional observability settings.""" + + @property + def sensitive_headers(self) -> set[str] | None: + """Set of header names to redact during logging.""" + ... + + @property + def metrics_registry(self) -> GrpcClientMetricsProtocol | None: + """Optional metrics registry for pool and client metrics.""" + ... + + +@runtime_checkable +class FullGrpcClientSettingsProtocol( + GrpcClientSettingsProtocol, + GrpcChannelExtrasProtocol, + GrpcObservabilityExtrasProtocol, + Protocol, +): + """Protocol for settings that carry the required fields and every optional block.""" + + +__all__ = [ + "ChannelPoolSettingsProtocol", + "ChannelProviderProtocol", + "CircuitBreakerMetricsProtocol", + "CircuitBreakerSettingsProtocol", + "FullGrpcClientSettingsProtocol", + "GrpcChannelExtrasProtocol", + "GrpcClientMetricsProtocol", + "GrpcClientSettingsProtocol", + "GrpcObservabilityExtrasProtocol", + "HealthCheckerProtocol", + "HealthCheckerSettingsProtocol", + "HealthStatusCallbackProtocol", + "LoadBalancerSettingsProtocol", + "RetryMetricsProtocol", + "RetrySettingsProtocol", + "TimeoutSettingsProtocol", +] diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 8887c3c..3fb3a65 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -53,6 +53,16 @@ def find_spec(self, name, path=None, target=None): assert "grpc-client-kit[health]" in str(exc), str(exc) else: raise AssertionError("HealthChecker must not resolve without the extra") + + # An ImportError rather than an AttributeError, on purpose: a missing extra is an install + # problem and must say so. hasattr() therefore propagates it instead of answering False, + # which is why the extra is probed for by name and not through the attribute. + try: + hasattr(grpc_client_kit, "HealthChecker") + except ImportError as exc: + assert "grpc-client-kit[health]" in str(exc), str(exc) + else: + raise AssertionError("hasattr must not swallow the missing extra") """ ) diff --git a/tests/unit/test_init.py b/tests/unit/test_init.py index b22c89c..92adc47 100644 --- a/tests/unit/test_init.py +++ b/tests/unit/test_init.py @@ -13,7 +13,9 @@ GrpcClientKitError, HealthCheckerNotRunningError, NoHealthyTargetsError, + protocols, ) +from grpc_client_kit.interceptors import circuit_breaker pytestmark = pytest.mark.unit @@ -76,6 +78,33 @@ def test__public_api__pool_internals__are_not_part_of_the_contract() -> None: assert "chain_token" not in grpc_client_kit.__all__ +def test__public_api__every_declared_protocol__is_re_exported_at_the_top_level() -> None: + """A settings object is written against these, so importing them must not need the submodule.""" + # Act + missing = sorted(set(protocols.__all__) - set(grpc_client_kit.__all__)) + + # Assert + assert not missing, f"declared in grpc_client_kit.protocols but not exported: {missing}" + + +def test__public_api__protocols_module__declares_every_protocol_the_package_exports() -> None: + """A protocol the package exports but the module does not declare is missed by a star import.""" + # Act + exported = {name for name in grpc_client_kit.__all__ if hasattr(protocols, name)} + undeclared = sorted(exported - set(protocols.__all__)) + + # Assert + assert not undeclared, f"exported by the package but absent from protocols.__all__: {undeclared}" + + +def test__public_api__circuit_breaker_internals__are_not_part_of_the_contract() -> None: + """``MethodCircuitState`` is the breaker's mutable bookkeeping; callers get status snapshots.""" + # Act & Assert + assert "MethodCircuitState" not in circuit_breaker.__all__ + assert "MethodCircuitState" not in grpc_client_kit.__all__ + assert "CircuitBreakerStatus" in circuit_breaker.__all__ + + def test__kit_errors__every_local_failure__is_catchable_as_one_family() -> None: """``except GrpcClientKitError`` is the one handler for "the kit said no, not the server".""" # Act & Assert