From 7790b26d5652f8574ffe14410080aecf3d44845c Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Fri, 30 Oct 2026 19:03:54 +0600 Subject: [PATCH] feat: Implement AllowlistCORSMiddleware for private network access control - Added AllowlistCORSMiddleware to restrict private network access to allowed origins. - Updated setup_middleware.py to use AllowlistCORSMiddleware instead of CORSMiddleware. - Enhanced rate limiting middleware to handle backend unavailability, returning 503 status with appropriate error message. - Introduced checks for backend availability during application startup, raising StartupCheckError if any configured backend is unreachable. - Created tests for new middleware and startup checks to ensure proper functionality and error handling. - Updated dependencies to version 1.0.0rc3. --- CHANGELOG.md | 33 +++ CLAUDE.md | 4 +- README.md | 2 +- RELEASE_NOTES.md | 28 +- SECURITY.md | 2 +- pyproject.toml | 10 +- tenxgraph_api/cli/config_editor/schema.py | 19 +- tenxgraph_api/cli/main.py | 2 +- .../references/api-settings-and-middleware.md | 10 + .../10xgraph/references/rate-limiting.md | 25 +- tenxgraph_api/src/app/core/config/settings.py | 13 + .../src/app/core/config/setup_middleware.py | 22 +- .../app/core/middleware/rate_limit/base.py | 23 +- .../core/middleware/rate_limit/middleware.py | 32 ++- .../app/core/middleware/rate_limit/redis.py | 41 ++- tenxgraph_api/src/app/main.py | 25 ++ .../src/app/routers/ping/readiness.py | 51 +++- tenxgraph_api/src/app/routers/ping/router.py | 12 +- .../src/app/routers/ping/startup_checks.py | 231 ++++++++++++++++ tests/unit_tests/test_middleware_order.py | 32 ++- .../unit_tests/test_rate_limit_unavailable.py | 117 +++++++++ tests/unit_tests/test_startup_checks.py | 247 ++++++++++++++++++ uv.lock | 14 +- 23 files changed, 941 insertions(+), 54 deletions(-) create mode 100644 tenxgraph_api/src/app/routers/ping/startup_checks.py create mode 100644 tests/unit_tests/test_rate_limit_unavailable.py create mode 100644 tests/unit_tests/test_startup_checks.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fdb5202b..e775ffe2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,39 @@ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). --- +## [1.0.0rc3] - 2026-10-30 + +The third release candidate of `10xgraph-api` 1.0. It requires `10xgraph` 1.0.0rc3 (stores have +`aping()`) and is the `1.0.0rc2` code plus the changes below. `10xgraph-client` stays at 1.0.0-rc.2. + +### Added + +- The server refuses to start while a configured backend is unreachable: the checkpointer's + Postgres (query and schema version) and Redis, the store (when it has `aping`/`ping`), the + API's Redis (`redis` in `10xgraph.json` or `REDIS_URL`) and the rate limiter's own Redis. It + retries with a short backoff for `STARTUP_CHECK_TIMEOUT_SECONDS` (default 30), so services + started together can come up in any order, then exits non-zero with each failed component, its + `host:port` (no credentials) and the error. This applies in production too, so a broken + connection string fails the deploy instead of the first request. `STARTUP_CHECKS=false` skips + it. Before, the server started anyway and, with a fail-closed Redis rate limiter, answered every + request except `/ping` with 429. + +### Changed + +- A rate-limiter backend outage with `fail_open: false` now answers `503` with error code + `RATE_LIMIT_UNAVAILABLE` and `Retry-After: 5` instead of `429 RATE_LIMIT_EXCEEDED`; 429 is kept + for a real limit hit. Clients that retried on 429 now also get 503 during a Redis outage and + should retry on it too. A custom backend can signal the same with + `RateLimitDecision.backend_unavailable()`; backends that return only allow/deny work unchanged. + The Redis backend logs a traceback for the first error of an outage and one line per error + after it, instead of a traceback per request. + +### Fixed + +- The hosted playground can reach a local `10xgraph dev` server: CORS answers the browser's + private-network preflight (`Access-Control-Request-Private-Network`) for origins in `ORIGINS`. + Before, it answered 400 and the browser reported a CORS error. + ## [1.0.0rc2] - 2026-10-30 The second release candidate of `10xgraph-api` 1.0. It requires `10xgraph` 1.0.0rc2 and is aligned diff --git a/CLAUDE.md b/CLAUDE.md index b4f81948..be7abf4a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,13 +5,13 @@ core framework see `agentflow/CLAUDE.md`; for the TS client, docs, or playground for the monorepo overview see the workspace-root `CLAUDE.md`. - Package name (PyPI): `10xgraph-api` (formerly `10xscale-agentflow-cli`, last release 0.6.x) -- Version: `1.0.0rc2` (`pyproject.toml`; 0.7.0 was the first release under the 10xGraph name). `CLI_VERSION` +- Version: `1.0.0rc3` (`pyproject.toml`; 0.7.0 was the first release under the 10xGraph name). `CLI_VERSION` and `tenxgraph_api.__version__` are single-sourced from the installed distribution metadata (falling back to `pyproject.toml` only for a non-installed source checkout). - Requires: Python >= 3.12 ยท Status: `5 - Production/Stable` - Console entry points: `10xgraph = tenxgraph_api.cli.main:main`, plus the deprecated alias `agentflow = tenxgraph_api.cli.main:legacy_main` (prints a notice to stderr; removed in 2.0) -- Depends on the core framework: `10xgraph>=1.0.0rc2,<2.0` (import `tenxgraph`). Must not be +- Depends on the core framework: `10xgraph>=1.0.0rc3,<2.0` (import `tenxgraph`). Must not be installed alongside the old `10xscale-agentflow`: both provide an `agentflow` module. ## Rename compatibility (kept until 2.0) diff --git a/README.md b/README.md index c9536538..2aaa9676 100644 --- a/README.md +++ b/README.md @@ -527,7 +527,7 @@ sdist + wheel, checks the distribution metadata, and creates a GitHub Release wi notes and the artifacts attached. PyPI publishing is manual (`make publish`). ```bash -git tag v1.0.0rc2 && git push origin v1.0.0rc2 +git tag v1.0.0rc3 && git push origin v1.0.0rc3 ``` --- diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 738cb7bf..9df9f7b9 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,4 +1,30 @@ -# 10xgraph-api 1.0.0rc2 +# 10xgraph-api 1.0.0rc3 + +The third 1.0 release candidate: the `1.0.0rc2` code plus a startup check for every configured backend, a clear 503 when the rate limiter's backend is down, and a CORS fix so the hosted playground can reach a local `10xgraph dev` server. + +```bash +pip install -U --pre "10xgraph-api==1.0.0rc3" +``` + +This pulls `10xgraph` 1.0.0rc3. `10xgraph-client` stays at 1.0.0-rc.2 (no client changes). + +## Before you upgrade (changes in rc3) + +- **The server refuses to start while a configured backend is unreachable**, in development and production. It checks the checkpointer's Postgres (query and schema version) and Redis, the memory store (`aping()`, now required in `10xgraph` 1.0.0rc3), the API's Redis and the rate limiter's Redis. It retries for `STARTUP_CHECK_TIMEOUT_SECONDS` (default 30), then exits naming each failed backend and its `host:port` (no credentials). Set `STARTUP_CHECKS=false` to skip it. +- **A rate-limiter backend outage answers 503, not 429.** With `fail_open: false` and Redis unreachable, requests get `503 RATE_LIMIT_UNAVAILABLE` with `Retry-After: 5`. A real limit hit is still `429 RATE_LIMIT_EXCEEDED`. Retry on 503 as well as 429. +- **Running `10xgraph dev` on your machine against Docker backends:** publish Postgres, Redis and Qdrant on `127.0.0.1` and use `localhost` in `.env`. The compose service names only resolve inside the Docker network, and the startup check now reports this at boot. + +## What changed in rc3 + +- The hosted playground (`https://play.10xgraph.com`) can reach a local `10xgraph dev` server: CORS answers the browser's private-network preflight for origins in `ORIGINS` (before, it answered 400 and the browser reported a CORS error). Chrome also asks the user to allow access to the local machine; the playground now shows the steps. +- A custom rate-limit backend can signal an outage with `RateLimitDecision.backend_unavailable()`. +- The Redis rate-limit backend logs one traceback per outage instead of one per request. + +See [CHANGELOG.md](CHANGELOG.md) for the full list. The rc2 notes follow. + +--- + +## 10xgraph-api 1.0.0rc2 The second 1.0 release candidate: it is the `1.0.0rc1` code plus graceful shutdown, stricter request validation, 409 answers for writes during a run, idempotency keys, JWT key rotation and a redesigned eval and config experience. diff --git a/SECURITY.md b/SECURITY.md index 2e56f013..8b1b8da9 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,7 +7,7 @@ | `1.0.x` (`10xgraph-api`) | Yes | | `< 1.0` | No | -`10xgraph-api` is at 1.0 (release candidate `1.0.0rc2`). Security fixes are applied to the +`10xgraph-api` is at 1.0 (release candidate `1.0.0rc3`). Security fixes are applied to the latest published release only; there are no long-term support branches. The old `10xscale-agentflow-cli` package is no longer published; upgrade to `10xgraph-api` to receive fixes. diff --git a/pyproject.toml b/pyproject.toml index 71d8f48b..eb9d53d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "10xgraph-api" -version = "1.0.0rc2" +version = "1.0.0rc3" description = "Production server for 10xGraph agents (formerly 10xscale-agentflow-cli). Serves a compiled graph over REST, SSE, WebSocket, realtime audio and AG-UI with JWT or custom auth, scoped authorization on every endpoint, owner-only threads and rate limiting, and generates Docker Compose and Kubernetes files. Includes the 10xgraph CLI. MIT licensed, self-hosted." readme = "README.md" license = "MIT" @@ -60,7 +60,7 @@ classifiers = [ dependencies = [ # This release needs core 1.0 (import `tenxgraph`). The explicit rc in the # specifier lets pip and uv pick the core pre-release until 1.0.0 is out. - "10xgraph>=1.0.0rc2,<2.0", + "10xgraph>=1.0.0rc3,<2.0", # Pre-1.0 and major-version-risky deps are capped; floors are the versions # the suite is verified against. "fastapi>=0.116,<1.0", @@ -105,7 +105,7 @@ media = [ # Pillow for image resize / EXIF / format conversion in core's media processor. # Without it, uploaded images pass through unchanged. images = [ - "10xgraph[images]>=1.0.0rc2,<2.0", + "10xgraph[images]>=1.0.0rc3,<2.0", ] gcloud = [ "google-cloud-logging", @@ -312,6 +312,8 @@ convention = "google" [tool.pytest.ini_options] env = [ "ENVIRONMENT=pytest", + # Tests build the app without real backends; the startup gate has its own tests. + "STARTUP_CHECKS=false", ] testpaths = [ "tests", @@ -374,7 +376,7 @@ ENVIRONMENT = "pytest" [dependency-groups] dev = [ - "10xgraph>=1.0.0rc2", + "10xgraph>=1.0.0rc3", "snowflakekit", "pytest>=8.4.2", "pytest-asyncio>=1.2.0", diff --git a/tenxgraph_api/cli/config_editor/schema.py b/tenxgraph_api/cli/config_editor/schema.py index ee2de8bf..093eee7e 100644 --- a/tenxgraph_api/cli/config_editor/schema.py +++ b/tenxgraph_api/cli/config_editor/schema.py @@ -301,7 +301,10 @@ "default": True, "label": "Fail open", "type": "bool", - "help": "Allow requests when the backend errors. Off means deny.", + "help": ( + "Allow requests when the backend errors. Off means deny them with 503 " + "RATE_LIMIT_UNAVAILABLE." + ), }, { "path": ["rate_limit", "trusted_proxy_headers"], @@ -949,6 +952,20 @@ "docs enabled. Unset means on in production; false only warns." ), }, + { + "name": "STARTUP_CHECKS", + "purpose": ( + "Refuse to start while a configured Postgres, Redis or store is unreachable, " + "in development and production. Defaults to true; false skips the check." + ), + }, + { + "name": "STARTUP_CHECK_TIMEOUT_SECONDS", + "purpose": ( + "Seconds startup keeps retrying an unreachable backend before it gives up. " + "Defaults to 30." + ), + }, ], }, { diff --git a/tenxgraph_api/cli/main.py b/tenxgraph_api/cli/main.py index 4cddc660..e29c4912 100644 --- a/tenxgraph_api/cli/main.py +++ b/tenxgraph_api/cli/main.py @@ -49,7 +49,7 @@ def execute(self, *args: Any, **kwargs: Any) -> int: hints=( "Run `10xgraph audit` to inspect installed package compatibility.", "Upgrade with `python -m pip install --upgrade " - '"10xgraph>=1.0.0rc2,<2"`.', + '"10xgraph>=1.0.0rc3,<2"`.', ), ) from exc implementation = getattr(module, class_name) diff --git a/tenxgraph_api/cli/templates/skills/10xgraph/references/api-settings-and-middleware.md b/tenxgraph_api/cli/templates/skills/10xgraph/references/api-settings-and-middleware.md index 4fc58702..95f23446 100644 --- a/tenxgraph_api/cli/templates/skills/10xgraph/references/api-settings-and-middleware.md +++ b/tenxgraph_api/cli/templates/skills/10xgraph/references/api-settings-and-middleware.md @@ -19,6 +19,8 @@ Important variables: - `DOCS_PATH` - `REDOCS_PATH` - `REDIS_URL` +- `STARTUP_CHECKS` +- `STARTUP_CHECK_TIMEOUT_SECONDS` - `SENTRY_DSN` - `JWT_SECRET_KEY` - `JWT_ALGORITHM` @@ -64,6 +66,14 @@ default in production (`STRICT_PRODUCTION_CHECKS=false` downgrades them to warni - docs endpoints enabled - `ALLOWED_HOST=*` +## Startup Checks + +In every mode, the server refuses to start while a configured backend is unreachable: the +checkpointer's Postgres and Redis, the store (when it has `aping`/`ping`), `REDIS_URL` (or +`redis` in `10xgraph.json`) and the rate limiter's Redis. It retries for +`STARTUP_CHECK_TIMEOUT_SECONDS` (30) first, then exits with each failed component, its +`host:port` and the error. `STARTUP_CHECKS=false` skips the check. + ## Sentry `SENTRY_DSN` enables Sentry setup through the API config module. Keep error reporting optional and safe when unset. diff --git a/tenxgraph_api/cli/templates/skills/10xgraph/references/rate-limiting.md b/tenxgraph_api/cli/templates/skills/10xgraph/references/rate-limiting.md index a94fa1a8..99f17229 100644 --- a/tenxgraph_api/cli/templates/skills/10xgraph/references/rate-limiting.md +++ b/tenxgraph_api/cli/templates/skills/10xgraph/references/rate-limiting.md @@ -78,7 +78,7 @@ single Redis operation, which prevents concurrent requests from racing past the | `trusted_proxy_headers` | boolean | `false` | Use `X-Forwarded-For` as the client IP (only behind a trusted proxy). | | `redis.url` | string | `null` | Redis URL; required for the Redis backend. Supports `${ENV_VAR}` expansion. | | `redis.prefix` | string | `"10xgraph:rate-limit"` | Prefix for all Redis keys. | -| `fail_open` | boolean | `true` | On Redis errors, allow (`true`) or deny (`false`) requests. | +| `fail_open` | boolean | `true` | On Redis errors, allow (`true`) or deny (`false`, 503) requests. | ## Identity Modes @@ -107,7 +107,7 @@ Every response includes: | `X-RateLimit-Remaining` | Requests remaining in the current window. | | `X-RateLimit-Reset` | Unix timestamp for the window reset estimate. | | `X-RateLimit-Reset-After` | Seconds until the window reset estimate. | -| `Retry-After` | Present on `429` responses only. | +| `Retry-After` | Present on `429` and `503` responses only. | When the limit is exceeded, 10xGraph returns `429 Too Many Requests`: @@ -127,6 +127,24 @@ When the limit is exceeded, 10xGraph returns `429 Too Many Requests`: } ``` +When the backend cannot be reached and `fail_open` is `false`, the request is denied with +`503 Service Unavailable` and `Retry-After: 5`, not 429, so clients can tell an outage from a +limit hit: + +```json +{ + "error": { + "code": "RATE_LIMIT_UNAVAILABLE", + "message": "Rate limiter unavailable: its backend could not be reached.", + "retry_after_seconds": 5 + }, + "metadata": { + "request_id": "request-id", + "status": "error" + } +} +``` + ## Custom Backend Implement `BaseRateLimitBackend` and bind the instance through InjectQ, then set @@ -154,6 +172,9 @@ class MyRateLimitBackend(BaseRateLimitBackend): return None ``` +When the backend's store cannot be reached and the request must not pass, return +`RateLimitDecision.backend_unavailable()`; the middleware answers it with the 503 above. + ```json { "rate_limit": { diff --git a/tenxgraph_api/src/app/core/config/settings.py b/tenxgraph_api/src/app/core/config/settings.py index 060fd8bd..31cd80ec 100644 --- a/tenxgraph_api/src/app/core/config/settings.py +++ b/tenxgraph_api/src/app/core/config/settings.py @@ -160,6 +160,19 @@ class Settings(BaseSettings): # ``redis_key_prefix`` option so its cache keys are separated too. REDIS_KEY_PREFIX: str = "" + ################################# + ###### Startup checks ########### + ################################# + # At startup, before serving, ping every configured backend (the checkpointer's Postgres + # and Redis, the store, the API's Redis, the rate limiter's Redis) and refuse to start when + # one stays unreachable. Applies in development and production, so a wrong connection + # string fails the boot (and the deploy) instead of the first request. False skips it. + STARTUP_CHECKS: bool = True + # How long, in seconds, startup keeps retrying a failing backend (with a short backoff) + # before it gives up, so services started together (docker compose) can come up in any + # order. Each attempt is bounded by the readiness probe's own per-check timeout. + STARTUP_CHECK_TIMEOUT_SECONDS: float = Field(default=30.0, ge=0.0) + ################################# ###### Shutdown ################# ################################# diff --git a/tenxgraph_api/src/app/core/config/setup_middleware.py b/tenxgraph_api/src/app/core/config/setup_middleware.py index fe863b4c..8290cef3 100644 --- a/tenxgraph_api/src/app/core/config/setup_middleware.py +++ b/tenxgraph_api/src/app/core/config/setup_middleware.py @@ -7,9 +7,10 @@ from fastapi.middleware.trustedhost import TrustedHostMiddleware from fastapi.responses import JSONResponse from injectq import InjectQ -from starlette.datastructures import MutableHeaders +from starlette.datastructures import Headers, MutableHeaders from starlette.middleware.cors import CORSMiddleware from starlette.requests import Request +from starlette.responses import Response from starlette.types import ASGIApp, Message, Receive, Scope, Send from tenxgraph_api.src.app.core.exceptions.server_errors import UnhandledErrorMiddleware @@ -76,6 +77,21 @@ def _is_health_check(scope: Scope) -> bool: StaleStateError = None # type: ignore[assignment] +class AllowlistCORSMiddleware(CORSMiddleware): + """``CORSMiddleware`` that grants private-network access only to allowed origins. + + Starlette adds ``Access-Control-Allow-Private-Network: true`` to every preflight that asks + for it, including one it refuses for its origin. That refusal still stands (400, no + ``Access-Control-Allow-Origin``), but an origin outside ``ORIGINS`` should not get the grant. + """ + + def preflight_response(self, request_headers: Headers) -> Response: + response = super().preflight_response(request_headers) + if not self.is_allowed_origin(origin=request_headers["origin"]): + del response.headers["Access-Control-Allow-Private-Network"] + return response + + class InsecureCorsConfigError(RuntimeError): """Raised when production is configured with wildcard CORS + credentials.""" @@ -507,11 +523,13 @@ def setup_middleware( app.add_middleware(RequestIDMiddleware) app.add_middleware( - CORSMiddleware, + AllowlistCORSMiddleware, allow_origins=origins, allow_credentials=allow_credentials, allow_methods=["*"], allow_headers=["*"], + # Answer the Private Network Access preflight the hosted playground sends to a local server. + allow_private_network=True, ) logger.debug("Middleware set up") diff --git a/tenxgraph_api/src/app/core/middleware/rate_limit/base.py b/tenxgraph_api/src/app/core/middleware/rate_limit/base.py index cd3ea4f5..0510a4d7 100644 --- a/tenxgraph_api/src/app/core/middleware/rate_limit/base.py +++ b/tenxgraph_api/src/app/core/middleware/rate_limit/base.py @@ -2,20 +2,39 @@ from dataclasses import dataclass +# Retry-After (seconds) sent with the 503 answered while the backend cannot be reached. +UNAVAILABLE_RETRY_AFTER_SECONDS = 5 + + @dataclass(frozen=True) class RateLimitDecision: - """Result of a single rate-limit check.""" + """Result of a single rate-limit check. + + ``unavailable`` marks a denial caused by the backend (it could not be reached), not by the + limit: the middleware then answers 503 ``RATE_LIMIT_UNAVAILABLE`` instead of 429, with + ``reset_after`` as its ``Retry-After``. Backends that never set it keep working unchanged. + """ allowed: bool remaining: int reset_after: int + unavailable: bool = False + + @classmethod + def backend_unavailable( + cls, retry_after: int = UNAVAILABLE_RETRY_AFTER_SECONDS + ) -> "RateLimitDecision": + """A denial because the backend failed; answered 503 with ``Retry-After``.""" + return cls(allowed=False, remaining=0, reset_after=retry_after, unavailable=True) class BaseRateLimitBackend(ABC): """Abstract base class for rate-limit backends. Users can implement this class and bind an instance into InjectQ when they - need a backend other than the built-in memory or Redis implementations. + need a backend other than the built-in memory or Redis implementations. A backend that + cannot reach its store and must not let the request through returns + :meth:`RateLimitDecision.backend_unavailable`. """ @abstractmethod diff --git a/tenxgraph_api/src/app/core/middleware/rate_limit/middleware.py b/tenxgraph_api/src/app/core/middleware/rate_limit/middleware.py index 6bba1d73..f687e727 100644 --- a/tenxgraph_api/src/app/core/middleware/rate_limit/middleware.py +++ b/tenxgraph_api/src/app/core/middleware/rate_limit/middleware.py @@ -10,7 +10,7 @@ from tenxgraph_api.src.app.core.config.graph_config import RateLimitConfig from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe -from .base import BaseRateLimitBackend +from .base import BaseRateLimitBackend, RateLimitDecision from .keying import client_key_for @@ -22,8 +22,9 @@ class RateLimitMiddleware: """Backend-agnostic rate-limit middleware. - Pure ASGI: it answers 429 itself or edits the ``http.response.start`` message on the way - out, so it adds no per-request task and never buffers a streaming body. + Pure ASGI: it answers 429 (limit hit) or 503 (backend unreachable) itself, or edits the + ``http.response.start`` message on the way out, so it adds no per-request task and never + buffers a streaming body. """ def __init__( @@ -54,6 +55,12 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: ) reset_at_epoch = int(time.time()) + decision.reset_after + if not decision.allowed and decision.unavailable: + # The backend failed and fails closed: an outage, not the client's fault. The + # backend logs the error itself, so this does not log per request. + await self._unavailable(decision, scope)(scope, receive, send) + return + if not decision.allowed: request_id = (scope.get("state") or {}).get("request_id", "unknown") logger.warning( @@ -107,3 +114,22 @@ async def send_with_headers(message: Message) -> None: await send(message) await self.app(scope, receive, send_with_headers) + + @staticmethod + def _unavailable(decision: RateLimitDecision, scope: Scope) -> JSONResponse: + retry_after = max(decision.reset_after, 1) + return JSONResponse( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + content={ + "error": { + "code": "RATE_LIMIT_UNAVAILABLE", + "message": "Rate limiter unavailable: its backend could not be reached.", + "retry_after_seconds": retry_after, + }, + "metadata": { + "request_id": (scope.get("state") or {}).get("request_id", "unknown"), + "status": "error", + }, + }, + headers={"Retry-After": str(retry_after)}, + ) diff --git a/tenxgraph_api/src/app/core/middleware/rate_limit/redis.py b/tenxgraph_api/src/app/core/middleware/rate_limit/redis.py index c8e060c5..78a90b0a 100644 --- a/tenxgraph_api/src/app/core/middleware/rate_limit/redis.py +++ b/tenxgraph_api/src/app/core/middleware/rate_limit/redis.py @@ -58,7 +58,17 @@ class RedisRateLimitBackend(BaseRateLimitBackend): - """Distributed sliding-window rate limiter backed by Redis.""" + """Distributed sliding-window rate limiter backed by Redis. + + On a Redis error it allows the request (``fail_open``) or denies it as unavailable, which + the middleware answers with 503, never with the 429 of a real limit hit. + """ + + # The URL the client was built from (``from_url``); None for a client passed in. The + # startup check pings it. + redis_url: str | None = None + # True while Redis keeps failing: only the first error of an outage logs a traceback. + _failing = False def __init__( self, @@ -88,7 +98,9 @@ def from_url( from tenxgraph_api.src.app.core.redis_client import create_redis redis = create_redis(redis_url) - return cls(redis=redis, prefix=prefix, fail_open=fail_open, close_redis=True) + backend = cls(redis=redis, prefix=prefix, fail_open=fail_open, close_redis=True) + backend.redis_url = redis_url + return backend async def check(self, key: str, *, limit: int, window: int) -> RateLimitDecision: redis_key = f"{self._prefix}:{key}" @@ -101,16 +113,33 @@ async def check(self, key: str, *, limit: int, window: int) -> RateLimitDecision keys=[redis_key], args=[now_ms, window_ms, limit, member], ) - return RateLimitDecision( + decision = RateLimitDecision( allowed=bool(result[0]), remaining=int(result[1]), reset_after=int(result[2]), ) - except Exception: - logger.exception("Redis rate-limit backend error") + except Exception as exc: + self._log_error(exc) if self._fail_open: return RateLimitDecision(allowed=True, remaining=0, reset_after=window) - return RateLimitDecision(allowed=False, remaining=0, reset_after=window) + return RateLimitDecision.backend_unavailable() + if self._failing: + self._failing = False + logger.info("Redis rate-limit backend recovered") + return decision + + def _log_error(self, exc: Exception) -> None: + """Traceback for the first error of an outage; one line for each error after it.""" + action = "allowing" if self._fail_open else "denying (503)" + if not self._failing: + self._failing = True + logger.exception( + "Redis rate-limit backend error; %s requests until it recovers", action + ) + else: + logger.warning( + "Redis rate-limit backend error (%s): %s: %s", action, type(exc).__name__, exc + ) async def close(self) -> None: if self._close_redis: diff --git a/tenxgraph_api/src/app/main.py b/tenxgraph_api/src/app/main.py index eec3b85e..f3b69c70 100644 --- a/tenxgraph_api/src/app/main.py +++ b/tenxgraph_api/src/app/main.py @@ -105,6 +105,28 @@ async def _prepare_checkpointer() -> None: ) +async def _check_backends(app: FastAPI) -> None: + """Refuse to start while a configured backend (Postgres, Redis, store) is unreachable. + + Retries for ``STARTUP_CHECK_TIMEOUT_SECONDS``, then raises ``StartupCheckError``, which + stops uvicorn and gunicorn with the failed components in the log. Off with + ``STARTUP_CHECKS=false``. + """ + if not settings.STARTUP_CHECKS: + logger.warning( + "STARTUP_CHECKS is off: starting without checking that the configured backends " + "are reachable." + ) + return + from tenxgraph_api.src.app.routers.ping.startup_checks import check_backends_at_startup + + await check_backends_at_startup( + container, + timeout=settings.STARTUP_CHECK_TIMEOUT_SECONDS, + rate_limit_backend=getattr(app.state, "rate_limit_backend", None), + ) + + async def _cleanup_temp_media_cache() -> None: """Run best-effort cleanup of expired temporary media cache entries.""" try: @@ -143,6 +165,9 @@ async def lifespan(app: FastAPI): await _prepare_checkpointer() + # Fail fast, in development and production, while a configured backend is unreachable. + await _check_backends(app) + # Clean up expired temporary media cache on startup try: await asyncio.wait_for(_cleanup_temp_media_cache(), BOOT_CLEANUP_TIMEOUT_SECONDS) diff --git a/tenxgraph_api/src/app/routers/ping/readiness.py b/tenxgraph_api/src/app/routers/ping/readiness.py index 3c7631e5..bd32a2ea 100644 --- a/tenxgraph_api/src/app/routers/ping/readiness.py +++ b/tenxgraph_api/src/app/routers/ping/readiness.py @@ -118,18 +118,51 @@ async def check_redis(url: str | None) -> None: await client.aclose() -async def run_check(name: str, check: Callable[[], Awaitable[None]]) -> str: +async def check_outcome( + check: Callable[[], Awaitable[None]], timeout: float = CHECK_TIMEOUT_SECONDS +) -> tuple[str, BaseException | None]: + """Run one time-boxed check: ``(ok | fail | skipped, the error when it failed)``. + + A timeout is reported as a ``TimeoutError``. Shared by the readiness probe and the startup + check (``startup_checks``), so both judge a component the same way. + """ try: - await asyncio.wait_for(check(), CHECK_TIMEOUT_SECONDS) + await asyncio.wait_for(check(), timeout) except _Skip: - return SKIPPED - except TimeoutError: + return SKIPPED, None + except Exception as exc: # a timeout too: TimeoutError + return FAIL, exc + return OK, None + + +async def run_check(name: str, check: Callable[[], Awaitable[None]]) -> str: + result, error = await check_outcome(check) + if isinstance(error, TimeoutError): logger.warning("Readiness check %s timed out after %ss", name, CHECK_TIMEOUT_SECONDS) - return FAIL - except Exception as exc: - logger.warning("Readiness check %s failed: %s", name, exc) - return FAIL - return OK + elif error is not None: + logger.warning("Readiness check %s failed: %s", name, error) + return result + + +def readiness_targets(container: Any) -> tuple[Any, Any, str | None]: + """``(checkpointer, store, redis_url)`` the checks run against, from the app's container. + + Redis is the URL the API's own clients use (run locks and caps, ownership cache, token + revocation): ``redis`` in 10xgraph.json, else ``REDIS_URL``. None for what is not set. + """ + from tenxgraph.storage.checkpointer import BaseCheckpointer + from tenxgraph.storage.store import BaseStore + + from tenxgraph_api.src.app.core.config.graph_config import GraphConfig + from tenxgraph_api.src.app.core.config.settings import get_settings + + graph_config = container.try_get(GraphConfig) if GraphConfig in container else None + redis_url = getattr(graph_config, "redis_url", None) or get_settings().REDIS_URL + return ( + container.try_get(BaseCheckpointer) if BaseCheckpointer in container else None, + container.try_get(BaseStore) if BaseStore in container else None, + redis_url if isinstance(redis_url, str) else None, + ) def redis_required(graph_config: Any) -> bool: diff --git a/tenxgraph_api/src/app/routers/ping/router.py b/tenxgraph_api/src/app/routers/ping/router.py index 95757881..35b6eb2b 100644 --- a/tenxgraph_api/src/app/routers/ping/router.py +++ b/tenxgraph_api/src/app/routers/ping/router.py @@ -43,14 +43,11 @@ async def ping_server( ) async def readiness_probe(request: Request): from injectq import InjectQ - from tenxgraph.storage.checkpointer import BaseCheckpointer - from tenxgraph.storage.store import BaseStore from tenxgraph_api.src.app.core.config.graph_config import GraphConfig - from tenxgraph_api.src.app.core.config.settings import get_settings from tenxgraph_api.src.app.core.utils.container import current_container - from .readiness import cached_readiness, is_ready, redis_required + from .readiness import cached_readiness, is_ready, readiness_targets, redis_required if getattr(request.app.state, "draining", False): # Shutting down: leave the load balancer's rotation before the sockets close. @@ -58,12 +55,7 @@ async def readiness_probe(request: Request): container: InjectQ = current_container() graph_config = container.try_get(GraphConfig) if GraphConfig in container else None - redis_url = getattr(graph_config, "redis_url", None) or get_settings().REDIS_URL - checks = await cached_readiness( - container.try_get(BaseCheckpointer) if BaseCheckpointer in container else None, - container.try_get(BaseStore) if BaseStore in container else None, - redis_url if isinstance(redis_url, str) else None, - ) + checks = await cached_readiness(*readiness_targets(container)) ready = is_ready(checks, redis_is_required=redis_required(graph_config)) return JSONResponse( status_code=200 if ready else 503, diff --git a/tenxgraph_api/src/app/routers/ping/startup_checks.py b/tenxgraph_api/src/app/routers/ping/startup_checks.py new file mode 100644 index 00000000..af068075 --- /dev/null +++ b/tenxgraph_api/src/app/routers/ping/startup_checks.py @@ -0,0 +1,231 @@ +"""Startup gate: refuse to serve while a configured backend cannot be reached. + +A server whose Redis, Postgres or vector store is down still answers ``/ping``, and before +this gate it served every other request with an error (a fail-closed rate limiter answered +429 to everything). The lifespan now runs the readiness probes (``readiness.py``) for each +configured backend before the app serves, retrying with a short backoff for +``STARTUP_CHECK_TIMEOUT_SECONDS`` so services started together can come up in any order. When +one still fails, :class:`StartupCheckError` stops the boot and names each component, its +``host:port`` (never credentials) and the error. ``STARTUP_CHECKS=false`` turns the gate off. + +Unlike readiness, a Redis outage is not tolerated here: a configured backend that is wrong at +boot is a deployment mistake, not a transient outage. +""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import urlsplit + +from tenxgraph_api.src.app.core.utils.log_sanitizer import redact_text + +from .readiness import ( + CHECK_TIMEOUT_SECONDS, + FAIL, + OK, + check_checkpointer, + check_outcome, + check_redis, + check_store, + readiness_targets, +) + + +logger = logging.getLogger("tenxgraph_api.startup") + +FIRST_RETRY_DELAY_SECONDS = 0.5 +MAX_RETRY_DELAY_SECONDS = 5.0 +HINT = "Start Redis/Postgres/Qdrant, or set STARTUP_CHECKS=false to skip this check." + + +class StartupCheckError(RuntimeError): + """A configured backend stayed unreachable for the whole startup window.""" + + +@dataclass +class Target: + """One configured backend to check: a name, its probe and where it lives.""" + + name: str + check: Callable[[], Awaitable[None]] + endpoints: list[str] = field(default_factory=list) + + +def host_port(url: Any) -> str | None: + """``host:port`` of a URL or DSN, without user, password, path or query; None if unknown.""" + if not isinstance(url, str) or "://" not in url: + return None + try: + parts = urlsplit(url) + host = parts.hostname + port = parts.port + except ValueError: + return None + if not host: + return None + if ":" in host: + host = f"[{host}]" + return f"{host}:{port}" if port else host + + +def _redis_client_endpoint(client: Any) -> str | None: + """``host:port`` a redis-py client connects to, from its pool's connection settings.""" + kwargs = getattr(getattr(client, "connection_pool", None), "connection_kwargs", None) + if not isinstance(kwargs, dict) or not kwargs.get("host"): + return None + return f"{kwargs['host']}:{kwargs.get('port', 6379)}" + + +def _checkpointer_endpoints(checkpointer: Any) -> list[str]: + """Best effort: the Postgres and Redis a ``PgCheckpointer`` uses.""" + found = [] + pool_config = getattr(checkpointer, "_pg_pool_config", None) + dsn = pool_config.get("postgres_dsn") if isinstance(pool_config, dict) else None + if (where := host_port(dsn)) is not None: + found.append(f"postgres {where}") + if (where := _redis_client_endpoint(getattr(checkpointer, "redis", None))) is not None: + found.append(f"redis {where}") + return found + + +def _store_endpoints(store: Any) -> list[str]: + """Best effort: the server a ``QdrantStore`` talks to (its client's REST address).""" + remote = getattr(getattr(store, "client", None), "_client", None) + where = host_port(getattr(remote, "rest_uri", None)) + return [where] if where else [] + + +def startup_targets(container: Any, rate_limit_backend: Any = None) -> list[Target]: + """Every configured backend, as readiness sees it, plus the rate limiter's own Redis. + + Components that are not configured are left out. The rate limiter's Redis is added only + when it was built from its own URL and that URL differs from the API's Redis. + """ + checkpointer, store, redis_url = readiness_targets(container) + targets = [] + if checkpointer is not None: + targets.append( + Target( + "checkpointer", + lambda: check_checkpointer(checkpointer), + _checkpointer_endpoints(checkpointer), + ) + ) + if store is not None: + targets.append(Target("store", lambda: check_store(store), _store_endpoints(store))) + if redis_url: + targets.append( + Target("redis", lambda: check_redis(redis_url), [host_port(redis_url) or "?"]) + ) + limiter_url = getattr(rate_limit_backend, "redis_url", None) + if isinstance(limiter_url, str) and limiter_url and limiter_url != redis_url: + targets.append( + Target( + "rate_limit redis", + lambda: check_redis(limiter_url), + [host_port(limiter_url) or "?"], + ) + ) + elif hasattr(rate_limit_backend, "redis_url") and not limiter_url: + logger.info( + "Startup check: the rate limiter uses a Redis client bound in the container; " + "it is not checked at startup." + ) + return targets + + +def _describe(target: Target) -> str: + return f"{target.name} ({', '.join(target.endpoints)})" if target.endpoints else target.name + + +def _error_text(error: BaseException | None, timeout: float) -> str: + if isinstance(error, TimeoutError) or error is None: + return f"timed out after {timeout}s" + text = str(error).strip() + return redact_text(f"{type(error).__name__}: {text}" if text else type(error).__name__) + + +async def wait_for_backends( + targets: list[Target], + *, + timeout: float, + attempt_timeout: float = CHECK_TIMEOUT_SECONDS, + sleep: Callable[[float], Awaitable[Any]] = asyncio.sleep, +) -> None: + """Check ``targets`` until all pass or ``timeout`` seconds run out; raise if any still fail. + + Each round checks the components still failing, concurrently, each bounded by + ``attempt_timeout``. There is always at least one round. A component with nothing to check + (a store without ``aping``/``ping``) is skipped with an INFO line. + """ + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + delay = FIRST_RETRY_DELAY_SECONDS + pending = list(targets) + errors: dict[str, str] = {} + attempt = 0 + while True: + attempt += 1 + outcomes = await asyncio.gather( + *(check_outcome(target.check, attempt_timeout) for target in pending) + ) + still_failing = [] + for target, (result, error) in zip(pending, outcomes, strict=True): + if result == OK: + errors.pop(target.name, None) + if attempt > 1: + logger.info("Startup check: %s is reachable", _describe(target)) + elif result == FAIL: + errors[target.name] = _error_text(error, attempt_timeout) + still_failing.append(target) + else: + logger.info("Startup check: %s has nothing to check; skipped", target.name) + pending = still_failing + if not pending: + return + remaining = deadline - loop.time() + for target in pending: + logger.warning( + "Startup check: %s is unreachable (attempt %d): %s%s", + _describe(target), + attempt, + errors[target.name], + f"; retrying for up to {remaining:.0f}s more" if remaining > 0 else "", + ) + if remaining <= 0: + break + await sleep(min(delay, remaining)) + delay = min(delay * 2, MAX_RETRY_DELAY_SECONDS) + + lines = [f" - {_describe(target)}: {errors[target.name]}" for target in pending] + raise StartupCheckError( + "\n".join( + [ + f"Refusing to start: {len(pending)} configured backend(s) unreachable after " + f"{timeout:g}s:", + *lines, + HINT, + ] + ) + ) + + +async def check_backends_at_startup( + container: Any, *, timeout: float, rate_limit_backend: Any = None +) -> None: + """Run the startup gate for the app's container; log and raise when it fails.""" + targets = startup_targets(container, rate_limit_backend) + if not targets: + logger.debug("Startup check: no backends configured") + return + try: + await wait_for_backends(targets, timeout=timeout) + except StartupCheckError as exc: + # The text as the format string: an argument would have its line breaks escaped. + logger.critical(str(exc)) + raise + logger.info("Startup check passed: %s", ", ".join(target.name for target in targets)) diff --git a/tests/unit_tests/test_middleware_order.py b/tests/unit_tests/test_middleware_order.py index e88d1e02..4cedcba0 100644 --- a/tests/unit_tests/test_middleware_order.py +++ b/tests/unit_tests/test_middleware_order.py @@ -52,10 +52,10 @@ def _assert_decorated(response): def test_cors_is_outermost(): stack = [m.cls for m in _app().user_middleware] # outermost first - assert stack[0] is CORSMiddleware + assert issubclass(stack[0], CORSMiddleware) names = [cls.__name__ for cls in stack] assert names == [ - "CORSMiddleware", + "AllowlistCORSMiddleware", "RequestIDMiddleware", "SecurityHeadersMiddleware", "HealthCheckAwareTrustedHostMiddleware", @@ -98,6 +98,34 @@ def test_preflight_is_answered_before_the_rate_limiter(path): assert client.get(path, headers={"Origin": ORIGIN}).status_code == 200 +PREFLIGHT = {"Access-Control-Request-Method": "GET"} +PRIVATE_NETWORK = {"Access-Control-Request-Private-Network": "true"} + + +def test_private_network_preflight_from_allowed_origin_is_granted(): + # The hosted playground calling a local dev server: browsers add this header. + headers = {"Origin": ORIGIN, **PREFLIGHT, **PRIVATE_NETWORK} + resp = TestClient(_app()).options("/", headers=headers) + assert resp.status_code == 200 + assert resp.headers.get("access-control-allow-origin") == ORIGIN + assert resp.headers.get("access-control-allow-private-network") == "true" + + +def test_private_network_preflight_from_other_origin_is_refused(): + headers = {"Origin": "https://evil.example.com", **PREFLIGHT, **PRIVATE_NETWORK} + resp = TestClient(_app()).options("/", headers=headers) + assert resp.status_code == 400 + assert "access-control-allow-origin" not in resp.headers + assert "access-control-allow-private-network" not in resp.headers + + +def test_preflight_without_private_network_header_is_unchanged(): + resp = TestClient(_app()).options("/", headers={"Origin": ORIGIN, **PREFLIGHT}) + assert resp.status_code == 200 + assert resp.headers.get("access-control-allow-origin") == ORIGIN + assert "access-control-allow-private-network" not in resp.headers + + def test_unhandled_exception_is_a_json_500_with_request_id_and_headers(): app = _app() diff --git a/tests/unit_tests/test_rate_limit_unavailable.py b/tests/unit_tests/test_rate_limit_unavailable.py new file mode 100644 index 00000000..f4f9b8be --- /dev/null +++ b/tests/unit_tests/test_rate_limit_unavailable.py @@ -0,0 +1,117 @@ +"""A rate-limiter backend outage answers 503 RATE_LIMIT_UNAVAILABLE, never 429.""" + +# ruff: noqa: S101, PLR2004, SLF001 + +import logging + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from tenxgraph_api.src.app.core.config.graph_config import RateLimitConfig +from tenxgraph_api.src.app.core.middleware.rate_limit import ( + BaseRateLimitBackend, + MemoryRateLimitBackend, + RateLimitDecision, + RateLimitMiddleware, + RedisRateLimitBackend, +) + + +def _config(**overrides) -> RateLimitConfig: + data = {"enabled": True, "backend": "memory", "requests": 2, "window": 60, "by": "ip"} + data.update(overrides) + return RateLimitConfig.from_dict(data) + + +def _broken_redis_backend(*, fail_open: bool) -> RedisRateLimitBackend: + async def failing_script(*, keys, args): + raise ConnectionError("Error 111 connecting to localhost:6379. Connection refused.") + + backend = object.__new__(RedisRateLimitBackend) + backend._prefix = "10xgraph:test" + backend._fail_open = fail_open + backend._script = failing_script + return backend + + +def _client(backend: BaseRateLimitBackend, **overrides) -> TestClient: + app = FastAPI() + app.add_middleware(RateLimitMiddleware, config=_config(**overrides), backend=backend) + + @app.get("/") + def root(): + return {"ok": True} + + return TestClient(app) + + +def test_backend_error_with_fail_open_false_answers_503(): + resp = _client(_broken_redis_backend(fail_open=False)).get("/") + assert resp.status_code == 503 + assert resp.headers["Retry-After"] == "5" + error = resp.json()["error"] + assert error["code"] == "RATE_LIMIT_UNAVAILABLE" + assert error["message"] == "Rate limiter unavailable: its backend could not be reached." + assert error["retry_after_seconds"] == 5 + + +def test_backend_error_with_fail_open_true_allows_the_request(): + resp = _client(_broken_redis_backend(fail_open=True)).get("/") + assert resp.status_code == 200 + assert resp.json() == {"ok": True} + + +def test_real_limit_still_answers_429(): + client = _client(MemoryRateLimitBackend()) + assert client.get("/").status_code == 200 + assert client.get("/").status_code == 200 + resp = client.get("/") + assert resp.status_code == 429 + assert resp.json()["error"]["code"] == "RATE_LIMIT_EXCEEDED" + assert "Retry-After" in resp.headers + + +def test_custom_backend_can_signal_unavailable(): + class Down(BaseRateLimitBackend): + async def check(self, key: str, *, limit: int, window: int) -> RateLimitDecision: + return RateLimitDecision.backend_unavailable(retry_after=7) + + async def close(self) -> None: + return None + + resp = _client(Down()).get("/") + assert resp.status_code == 503 + assert resp.headers["Retry-After"] == "7" + + +def test_plain_denial_from_a_custom_backend_is_still_429(): + class Deny(BaseRateLimitBackend): + async def check(self, key: str, *, limit: int, window: int) -> RateLimitDecision: + return RateLimitDecision(allowed=False, remaining=0, reset_after=3) + + async def close(self) -> None: + return None + + resp = _client(Deny()).get("/") + assert resp.status_code == 429 + assert resp.headers["Retry-After"] == "3" + + +@pytest.mark.asyncio +async def test_outage_logs_one_traceback_then_one_line_per_error(caplog): + backend = _broken_redis_backend(fail_open=False) + with caplog.at_level(logging.INFO, logger="tenxgraph_api.rate_limit"): + for _ in range(3): + decision = await backend.check("client", limit=2, window=60) + assert decision.unavailable is True + records = [r for r in caplog.records if r.name == "tenxgraph_api.rate_limit"] + assert [r.exc_info is not None for r in records] == [True, False, False] + assert "Connection refused" in records[1].getMessage() + + async def working_script(*, keys, args): + return [1, 1, 60] + + backend._script = working_script + assert (await backend.check("client", limit=2, window=60)).allowed is True + assert "recovered" in caplog.records[-1].getMessage() diff --git a/tests/unit_tests/test_startup_checks.py b/tests/unit_tests/test_startup_checks.py new file mode 100644 index 00000000..a54f11d1 --- /dev/null +++ b/tests/unit_tests/test_startup_checks.py @@ -0,0 +1,247 @@ +"""The startup gate refuses to boot while a configured backend is unreachable.""" + +# ruff: noqa: S101, SLF001 + +import asyncio +import logging +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from fastapi import FastAPI +from injectq import InjectQ +from tenxgraph.storage.checkpointer import BaseCheckpointer +from tenxgraph.storage.store import BaseStore + +from tenxgraph_api.src.app import main as app_main +from tenxgraph_api.src.app.core import redis_client +from tenxgraph_api.src.app.core.config import settings as settings_mod +from tenxgraph_api.src.app.routers.ping import startup_checks +from tenxgraph_api.src.app.routers.ping.startup_checks import ( + StartupCheckError, + Target, + check_backends_at_startup, + host_port, + startup_targets, + wait_for_backends, +) + + +REDIS_URL = "redis://:s3cret-pass@127.0.0.1:6399/0" + + +class _FakeRedis: + def __init__(self, error: Exception | None = None, host="cache", port=6379): + self.error = error + self.connection_pool = SimpleNamespace( + connection_kwargs={"host": host, "port": port, "password": "s3cret-pass"} + ) + + async def ping(self): + if self.error: + raise self.error + return True + + async def aclose(self): + return None + + +class _FakePgCheckpointer: + """Looks like PgCheckpointer to the readiness probe.""" + + def __init__(self, error: Exception | None = None): + self.error = error + self._pg_pool_config = {"postgres_dsn": "postgresql://app:s3cret-pass@db:5432/app"} + self.redis = _FakeRedis() + + async def _run_query(self, fn): + if self.error: + raise self.error + return 1 + + +class _PingStore: + def __init__(self): + self.pings = 0 + + async def aping(self): + self.pings += 1 + + +class _NoPingStore: + pass + + +def _container(monkeypatch, *, checkpointer=None, store=None, redis_url=None) -> InjectQ: + monkeypatch.setattr(settings_mod, "get_settings", lambda: SimpleNamespace(REDIS_URL=redis_url)) + container = InjectQ() + if checkpointer is not None: + container.bind_instance(BaseCheckpointer, checkpointer) + if store is not None: + container.bind_instance(BaseStore, store) + return container + + +def _unreachable_redis(monkeypatch): + error = ConnectionError("Error 111 connecting to 127.0.0.1:6399. Connection refused.") + monkeypatch.setattr(redis_client, "create_redis", lambda url: _FakeRedis(error)) + + +async def _no_sleep(_delay): + await asyncio.sleep(0) + + +# ---------------------------------------------------------------- endpoints + + +def test_host_port_drops_credentials_path_and_query(): + assert host_port(REDIS_URL) == "127.0.0.1:6399" + assert host_port("postgresql://u:p@db.internal:5432/app?sslmode=require") == "db.internal:5432" + assert host_port("redis://[::1]:6379") == "[::1]:6379" + assert host_port("not a url") is None + assert host_port(None) is None + + +def test_unconfigured_components_are_left_out(monkeypatch): + assert startup_targets(_container(monkeypatch)) == [] + names = [t.name for t in startup_targets(_container(monkeypatch, store=_PingStore()))] + assert names == ["store"] + + +def test_rate_limiter_redis_is_checked_only_when_it_has_its_own_url(monkeypatch): + container = _container(monkeypatch, redis_url=REDIS_URL) + same = SimpleNamespace(redis_url=REDIS_URL) + other = SimpleNamespace(redis_url="redis://limits:6380/1") + bound_client = SimpleNamespace(redis_url=None) + assert [t.name for t in startup_targets(container, same)] == ["redis"] + assert [t.name for t in startup_targets(container, other)] == ["redis", "rate_limit redis"] + assert [t.name for t in startup_targets(container, bound_client)] == ["redis"] + + +# ---------------------------------------------------------------- gate + + +@pytest.mark.asyncio +async def test_boot_check_passes_when_every_backend_answers(monkeypatch): + monkeypatch.setattr(redis_client, "create_redis", lambda url: _FakeRedis()) + store = _PingStore() + container = _container( + monkeypatch, checkpointer=_FakePgCheckpointer(), store=store, redis_url=REDIS_URL + ) + await check_backends_at_startup(container, timeout=0) + assert store.pings == 1 + + +@pytest.mark.asyncio +async def test_boot_check_fails_naming_component_and_host_without_password(monkeypatch, caplog): + _unreachable_redis(monkeypatch) + container = _container(monkeypatch, redis_url=REDIS_URL) + with caplog.at_level(logging.WARNING, logger="tenxgraph_api.startup"): + with pytest.raises(StartupCheckError) as info: + await check_backends_at_startup(container, timeout=0) + message = str(info.value) + assert "redis (127.0.0.1:6399)" in message + assert "Connection refused" in message + assert "STARTUP_CHECKS=false" in message + assert "s3cret" not in message + assert "s3cret" not in caplog.text + assert any(r.levelno == logging.CRITICAL for r in caplog.records) + assert any("attempt 1" in r.getMessage() for r in caplog.records) + + +@pytest.mark.asyncio +async def test_failed_checkpointer_names_postgres_and_redis_hosts(monkeypatch): + error = ConnectionRefusedError("Connect call failed ('db', 5432)") + container = _container(monkeypatch, checkpointer=_FakePgCheckpointer(error)) + with pytest.raises(StartupCheckError) as info: + await check_backends_at_startup(container, timeout=0) + message = str(info.value) + assert "checkpointer (postgres db:5432, redis cache:6379)" in message + assert "ConnectionRefusedError" in message + assert "s3cret" not in message + + +@pytest.mark.asyncio +async def test_check_that_recovers_within_the_window_passes(monkeypatch): + calls = 0 + + async def flaky(): + nonlocal calls + calls += 1 + if calls < 3: + raise ConnectionError("not up yet") + + await wait_for_backends([Target("redis", flaky, ["r:6379"])], timeout=30, sleep=_no_sleep) + assert calls == 3 + + +@pytest.mark.asyncio +async def test_check_keeps_failing_until_the_window_closes(monkeypatch): + calls = 0 + loop = asyncio.get_running_loop() + clock = [loop.time()] + monkeypatch.setattr(loop, "time", lambda: clock[0]) + + async def down(): + nonlocal calls + calls += 1 + raise ConnectionError("down") + + async def advance(delay): + clock[0] += delay + + with pytest.raises(StartupCheckError, match="after 3s"): + await wait_for_backends([Target("store", down)], timeout=3, sleep=advance) + # 0.5 + 1 + 1.5 (the rest of the window): four rounds, then it gives up. + assert calls == 4 + + +@pytest.mark.asyncio +async def test_each_attempt_is_bounded_by_the_per_check_timeout(): + async def hang(): + await asyncio.sleep(30) + + with pytest.raises(StartupCheckError, match="timed out after 0.01s"): + await wait_for_backends([Target("store", hang)], timeout=0, attempt_timeout=0.01) + + +@pytest.mark.asyncio +async def test_store_without_ping_is_skipped_with_info(monkeypatch, caplog): + container = _container(monkeypatch, store=_NoPingStore()) + with caplog.at_level(logging.INFO, logger="tenxgraph_api.startup"): + await check_backends_at_startup(container, timeout=0) + assert "store has nothing to check; skipped" in caplog.text + + +# ---------------------------------------------------------------- lifespan + + +def _lifespan_patches(monkeypatch, container): + monkeypatch.setattr(app_main, "container", container) + monkeypatch.setattr(app_main, "attach_all_modules", AsyncMock(return_value=None)) + monkeypatch.setattr(app_main, "_cleanup_temp_media_cache", AsyncMock()) + monkeypatch.setattr(app_main.settings, "STARTUP_CHECK_TIMEOUT_SECONDS", 0.0) + + +@pytest.mark.asyncio +async def test_lifespan_refuses_to_start_when_redis_is_unreachable(monkeypatch): + _unreachable_redis(monkeypatch) + _lifespan_patches(monkeypatch, _container(monkeypatch, redis_url=REDIS_URL)) + monkeypatch.setattr(app_main.settings, "STARTUP_CHECKS", True) + with pytest.raises(StartupCheckError, match=r"redis \(127\.0\.0\.1:6399\)"): + async with app_main.lifespan(FastAPI()): + pytest.fail("the app must not start serving") + + +@pytest.mark.asyncio +async def test_startup_checks_false_skips_the_gate(monkeypatch, caplog): + _unreachable_redis(monkeypatch) + _lifespan_patches(monkeypatch, _container(monkeypatch, redis_url=REDIS_URL)) + monkeypatch.setattr(app_main.settings, "STARTUP_CHECKS", False) + gate = AsyncMock() + monkeypatch.setattr(startup_checks, "check_backends_at_startup", gate) + with caplog.at_level(logging.WARNING, logger="tenxgraph_api"): + async with app_main.lifespan(FastAPI()): + pass + gate.assert_not_called() + assert "STARTUP_CHECKS is off" in caplog.text diff --git a/uv.lock b/uv.lock index 1d95f40e..182e1816 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ [[package]] name = "10xgraph" -version = "1.0.0rc2" +version = "1.0.0rc3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "injectq" }, @@ -18,9 +18,9 @@ dependencies = [ { name = "python-dotenv" }, { name = "pyyaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/dc/99/3c2e5c419d4630688996f1598ad026f8da1d2eee1ee635d43183deedd6ab/10xgraph-1.0.0rc2.tar.gz", hash = "sha256:948a1c0f8ebe18689f0cd1a9c6832e3bfe708f12f15c1f8bf20a3832bc9fc72b", size = 685744, upload-time = "2026-10-30T09:54:04.821Z" } +sdist = { url = "https://files.pythonhosted.org/packages/93/97/37e955f09e3f1147944752f53e8d957f90912be3cd0b2899c09b2252f7b8/10xgraph-1.0.0rc3.tar.gz", hash = "sha256:36e7562b4391db133be57b7237c8988d9e991f7758b3a6341dde9d5b4821846c", size = 686432, upload-time = "2026-10-30T13:00:31.782Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/33/13b8713f1e254dd2e70ad7b0481535ed6f899e607eddaddb04b56d77c493/10xgraph-1.0.0rc2-py3-none-any.whl", hash = "sha256:bb5d713b9e97355b6281f346da446f694edb8cf2dd352f3653cf56f186193c2e", size = 742180, upload-time = "2026-10-30T09:54:01.963Z" }, + { url = "https://files.pythonhosted.org/packages/46/9b/b0d455b1ad8e9e2dd9e25d6320663b178f30553e70ee36e165fd2303d28f/10xgraph-1.0.0rc3-py3-none-any.whl", hash = "sha256:609ea1f9f65d5fda866eab5f0cd5fca5e3d41e848f6741540dd54646aad5a843", size = 742737, upload-time = "2026-10-30T13:00:28.974Z" }, ] [package.optional-dependencies] @@ -30,7 +30,7 @@ images = [ [[package]] name = "10xgraph-api" -version = "1.0.0rc2" +version = "1.0.0rc3" source = { editable = "." } dependencies = [ { name = "10xgraph" }, @@ -109,8 +109,8 @@ dev = [ [package.metadata] requires-dist = [ - { name = "10xgraph", specifier = ">=1.0.0rc2,<2.0" }, - { name = "10xgraph", extras = ["images"], marker = "extra == 'images'", specifier = ">=1.0.0rc2,<2.0" }, + { name = "10xgraph", specifier = ">=1.0.0rc3,<2.0" }, + { name = "10xgraph", extras = ["images"], marker = "extra == 'images'", specifier = ">=1.0.0rc3,<2.0" }, { name = "ag-ui-protocol", marker = "extra == 'ag-ui'", specifier = ">=1.0,<2" }, { name = "fastapi", specifier = ">=0.116,<1.0" }, { name = "firebase-admin", marker = "extra == 'firebase'", specifier = ">=6.5.0" }, @@ -140,7 +140,7 @@ provides-extras = ["sentry", "firebase", "snowflakekit", "redis", "jwt", "media" [package.metadata.requires-dev] dev = [ - { name = "10xgraph", specifier = ">=1.0.0rc2" }, + { name = "10xgraph", specifier = ">=1.0.0rc3" }, { name = "ag-ui-protocol", specifier = ">=1.0,<2" }, { name = "gunicorn", specifier = ">=23,<27" }, { name = "httpx", specifier = "==0.27.0" },