From a25e63c147064918ad20b2e396cac84900727df5 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 22 Sep 2026 03:33:48 +0400 Subject: [PATCH 01/12] Rebuild observability runtime for API v1 --- README.md | 392 +++--- changelog.d/29.breaking.md | 6 + deploy/gcp/README.md | 181 +++ deploy/gcp/alerts.yaml | 42 + deploy/gcp/collector/Dockerfile | 6 + deploy/gcp/collector/config.yaml | 71 + deploy/gcp/collector/service.yaml | 45 + deploy/gcp/create_alerts.py | 250 ++++ deploy/gcp/dashboard.json | 183 +++ deploy/gcp/iam.yaml | 36 + deploy/gcp/log-routing.yaml | 37 + deploy/gcp/verify.sh | 47 + deploy/gcp/verify_modal_wif.py | 194 +++ deploy/gcp/verify_otel.py | 253 ++++ deploy/gcp/workload-inventory.yaml | 70 + docs/operations/api-v1-observability.md | 153 +++ .../operations/google-cloud-stage3-runbook.md | 195 --- policyengine_observability/__init__.py | 206 +-- policyengine_observability/_metrics.py | 188 --- policyengine_observability/_operations.py | 428 ------ policyengine_observability/_requests.py | 291 ---- policyengine_observability/_state.py | 65 - policyengine_observability/_tracing.py | 434 ------ .../adapters/__init__.py | 5 + .../adapters/fastapi.py | 339 ++--- policyengine_observability/adapters/flask.py | 151 +- policyengine_observability/config.py | 588 ++++---- policyengine_observability/context.py | 270 ---- policyengine_observability/delivery.py | 272 ++++ .../destinations/__init__.py | 19 - .../destinations/base.py | 121 -- .../destinations/google_cloud_logging.py | 241 ---- .../destinations/manager.py | 223 --- .../destinations/queued.py | 300 ---- .../destinations/registry.py | 57 - .../destinations/stdout.py | 138 -- policyengine_observability/diagnostics.py | 98 ++ .../{destinations => }/google_credentials.py | 66 +- .../integrations/__init__.py | 4 + .../integrations/httpx.py | 55 +- policyengine_observability/logging.py | 380 ------ policyengine_observability/otel.py | 701 ++++++++++ policyengine_observability/runtime.py | 1209 +++++++++++------ policyengine_observability/schema.py | 197 +++ policyengine_observability/segments.py | 432 ------ pyproject.toml | 47 +- tests/conftest.py | 78 +- tests/fakes.py | 121 -- tests/runtime_helpers.py | 198 --- tests/test_adapters.py | 167 +++ tests/test_config_schema.py | 198 +++ tests/test_delivery.py | 276 ++++ tests/test_deployment_assets.py | 55 + tests/test_diagnostics_public_api.py | 118 ++ tests/test_fastapi_adapter.py | 380 ------ tests/test_flask_adapter.py | 171 --- tests/test_google_credentials.py | 346 +---- tests/test_google_destination.py | 334 ----- tests/test_log_profiles.py | 292 ---- tests/test_otel.py | 614 +++++++++ tests/test_public_api.py | 126 -- tests/test_queued_destination.py | 472 ------- tests/test_release_scripts.py | 76 -- tests/test_runtime.py | 291 ++++ tests/test_runtime_emission.py | 425 ------ tests/test_runtime_operations.py | 295 ---- tests/test_runtime_request_failures.py | 342 ----- tests/test_runtime_requests.py | 331 ----- tests/test_runtime_segments.py | 303 ----- tests/test_runtime_tracing.py | 420 ------ tests/test_stdout_destinations.py | 362 ----- uv.lock | 317 +++-- 72 files changed, 6393 insertions(+), 10401 deletions(-) create mode 100644 changelog.d/29.breaking.md create mode 100644 deploy/gcp/README.md create mode 100644 deploy/gcp/alerts.yaml create mode 100644 deploy/gcp/collector/Dockerfile create mode 100644 deploy/gcp/collector/config.yaml create mode 100644 deploy/gcp/collector/service.yaml create mode 100644 deploy/gcp/create_alerts.py create mode 100644 deploy/gcp/dashboard.json create mode 100644 deploy/gcp/iam.yaml create mode 100644 deploy/gcp/log-routing.yaml create mode 100644 deploy/gcp/verify.sh create mode 100644 deploy/gcp/verify_modal_wif.py create mode 100644 deploy/gcp/verify_otel.py create mode 100644 deploy/gcp/workload-inventory.yaml create mode 100644 docs/operations/api-v1-observability.md delete mode 100644 docs/operations/google-cloud-stage3-runbook.md delete mode 100644 policyengine_observability/_metrics.py delete mode 100644 policyengine_observability/_operations.py delete mode 100644 policyengine_observability/_requests.py delete mode 100644 policyengine_observability/_state.py delete mode 100644 policyengine_observability/_tracing.py delete mode 100644 policyengine_observability/context.py create mode 100644 policyengine_observability/delivery.py delete mode 100644 policyengine_observability/destinations/__init__.py delete mode 100644 policyengine_observability/destinations/base.py delete mode 100644 policyengine_observability/destinations/google_cloud_logging.py delete mode 100644 policyengine_observability/destinations/manager.py delete mode 100644 policyengine_observability/destinations/queued.py delete mode 100644 policyengine_observability/destinations/registry.py delete mode 100644 policyengine_observability/destinations/stdout.py create mode 100644 policyengine_observability/diagnostics.py rename policyengine_observability/{destinations => }/google_credentials.py (75%) delete mode 100644 policyengine_observability/logging.py create mode 100644 policyengine_observability/otel.py create mode 100644 policyengine_observability/schema.py delete mode 100644 policyengine_observability/segments.py delete mode 100644 tests/fakes.py delete mode 100644 tests/runtime_helpers.py create mode 100644 tests/test_adapters.py create mode 100644 tests/test_config_schema.py create mode 100644 tests/test_delivery.py create mode 100644 tests/test_deployment_assets.py create mode 100644 tests/test_diagnostics_public_api.py delete mode 100644 tests/test_fastapi_adapter.py delete mode 100644 tests/test_flask_adapter.py delete mode 100644 tests/test_google_destination.py delete mode 100644 tests/test_log_profiles.py create mode 100644 tests/test_otel.py delete mode 100644 tests/test_public_api.py delete mode 100644 tests/test_queued_destination.py delete mode 100644 tests/test_release_scripts.py create mode 100644 tests/test_runtime.py delete mode 100644 tests/test_runtime_emission.py delete mode 100644 tests/test_runtime_operations.py delete mode 100644 tests/test_runtime_request_failures.py delete mode 100644 tests/test_runtime_requests.py delete mode 100644 tests/test_runtime_segments.py delete mode 100644 tests/test_runtime_tracing.py delete mode 100644 tests/test_stdout_destinations.py diff --git a/README.md b/README.md index 3db5dde..455b945 100644 --- a/README.md +++ b/README.md @@ -1,188 +1,262 @@ # policyengine-observability -Shared PolicyEngine observability runtime for fail-open local timings, -structured logs, OpenTelemetry traces, and OpenTelemetry metrics. +`policyengine-observability` provides structured application logs, +OpenTelemetry traces and metrics, request propagation, and framework request +instrumentation for PolicyEngine services. -The package intentionally keeps framework support in adapters: +Version 2 has one explicit ownership model: `configure(config)` returns a +runtime, and every adapter or manual operation receives that runtime. The +package does not install a mutable global runtime, infer a remote destination, +or contact a remote service during configuration. -- `policyengine_observability.adapters.flask` -- `policyengine_observability.adapters.fastapi` -- `policyengine_observability.integrations.httpx` +## Install -OpenTelemetry support is installed and enabled by default. Timing and -structured logging run even without an OTLP collector; when no endpoint is -configured, spans and metrics stay in-process while logs still receive trace -context. Set `OTEL_ENABLED=false` to opt out. Configure -`OTEL_EXPORTER_OTLP_ENDPOINT` to export traces and metrics. +Install only the integrations used by the service: -## Log routing profiles +```bash +pip install "policyengine-observability[otel,otlp-grpc]" +pip install "policyengine-observability[flask,httpx,google]" +pip install "policyengine-observability[fastapi,httpx,google]" +``` + +The base package has no required dependencies. OpenTelemetry, OTLP exporters, +Google authentication and logging, and web frameworks are optional extras and +are imported only when configured or called. + +## Configure a runtime + +Service and deployment identity are always explicit. Standard OpenTelemetry +environment variables may supply OTLP transport settings. + +```python +from policyengine_observability import ( + DeploymentIdentity, + ObservabilityConfig, + ServiceIdentity, + configure, +) + +config = ObservabilityConfig.from_env( + service=ServiceIdentity( + name="policyengine-api", + namespace="policyengine.api-v1", + version="1.2.3", + role="api", + ), + deployment=DeploymentIdentity( + environment="production", + platform="google_cloud_run", + region="us-central1", + ), + google_cloud_project_id="policyengine-observability", +) +runtime = configure(config) +``` + +Cloud Run writes one-line JSON to standard output. It does not use the Cloud +Logging API from the application process. When no OTLP endpoint is configured, +the runtime retains local trace context for log correlation and creates no +remote exporter. -Log routing is owned by this package: consumers set one env var and the -package expands it into destinations, formats, and transport. +The supported OTel environment settings include: ```bash -OBSERVABILITY_LOG_PROFILE=gcp-agent # or gcp-direct | plain-sync | auto +OTEL_EXPORTER_OTLP_ENDPOINT=https://COLLECTOR_HOST +OTEL_EXPORTER_OTLP_PROTOCOL=grpc +OTEL_TRACES_SAMPLER_ARG=1.0 +POLICYENGINE_OTEL_GOOGLE_AUDIENCE=https://COLLECTOR_HOST ``` -- `gcp-agent` — google-format stdout only, for platforms whose logging - agent ingests stdout (Cloud Run, GKE). Fully synchronous, zero - threads; the agent ships lines to Cloud Logging with severity, trace, - span, and labels promoted to first-class LogEntry fields. -- `gcp-direct` — plain stdout (the durable record) plus queued direct - Cloud Logging writes, for platforms with no ingesting agent (Modal). - Requires a resolvable Google Cloud project; downgrades to `plain-sync` - with a warning otherwise. -- `plain-sync` — plain stdout only, guaranteed zero threads. Local - development and the kill switch: setting it disables all background - log machinery. -- `auto` (default) — detects the platform via `OBSERVABILITY_PLATFORM` - (`google_cloud_run`/`modal`), then `K_SERVICE`, then Modal env - markers; when nothing matches, caller-supplied defaults apply. - -The granular controls still exist underneath and override the profile's -expansion when set explicitly: +## Flask and FastAPI -```bash -OBSERVABILITY_LOG_DESTINATIONS=stdout,google_cloud_logging -OBSERVABILITY_STDOUT_FORMAT=google -OBSERVABILITY_GOOGLE_CLOUD_PROJECT=policyengine-api -OBSERVABILITY_GOOGLE_CLOUD_LOG_NAME=policyengine-observability +Install a framework adapter once while constructing the application. Routes +do not need observability decorators. + +```python +from flask import Flask +from policyengine_observability import instrument_flask + +app = Flask(__name__) +instrument_flask(app, runtime) ``` -Destinations are named strategies: `stdout` is `inline` (synchronous on -the caller's thread), and every `remote` strategy — `google_cloud_logging` -today; future backends register the same way — is automatically wrapped -in the queued transport below. Google Cloud Logging uses Application -Default Credentials and requires permission to create log entries, -typically through `roles/logging.logWriter`. - -Backends plug in through two top-level hooks, `register_destination` -(with `transport="inline"|"remote"` and an optional `required_config` -tuple naming the config fields the strategy needs — profiles that name -the strategy downgrade gracefully when one is missing) and -`register_stdout_formatter`. Registration happens at import time, so an -external backend module must be imported before observability is -configured. Backend-specific knobs are the strategy's own: the Google -strategy reads `OBSERVABILITY_GOOGLE_WRITE_TIMEOUT_SECONDS` itself at -construction (so it is re-read on `restart_observability()`) rather -than through a core config field. Name lookups for destinations, -formatters, and profiles all forgive case, whitespace, and -hyphen/underscore variance; an unknown format name falls back to plain -and is reported through the internal-error channel, and a registered -formatter factory that raises degrades to the built-in plain formatter -the same way rather than breaking configuration. - -## Log emission and delivery semantics - -Remote destinations never write on a request thread. The log call only -snapshots the payload, stamps the enqueue time, and appends to a bounded -in-memory queue (microseconds, never blocks, never raises); a stdlib -`QueueListener` thread drains the queue and performs the writes, sending -the enqueue time as the entry timestamp so delayed writes keep their -event time. - -Delivery through the queue is best-effort by design — stdout is the -durable sibling record. When the queue is full the newest record is -dropped, and drops are counted and reported through the internal-error -channel (first drop, then every 100th). Write failures are likewise -reported and the record dropped; there is deliberately no breaker or -retry queue in the transport. Each Google write carries an explicit -budget that caps the call and its transient-error retries: +```python +from fastapi import FastAPI +from policyengine_observability import instrument_fastapi -```bash -OBSERVABILITY_GOOGLE_WRITE_TIMEOUT_SECONDS=10.0 -OBSERVABILITY_LOG_QUEUE_MAXSIZE=1000 -OBSERVABILITY_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS=2.0 +app = FastAPI() +instrument_fastapi(app, runtime) ``` -(The bounded write rebinds the client's private gapic method at -construction; when that handle is unavailable — HTTP transports, -injected fakes — the library's ~60s default applies, which is harmless -off the request path. All numeric knobs are clamped, so `0`, negative, -or non-finite values can never disable or unbound a mechanism. The -Google client library's one-time instrumentation diagnostic entry is -suppressed at construction, so the stream carries only the records the -service asked to write.) - -Shutdown closes log destinations inside the same bounded budget that -flushes OpenTelemetry (`OBSERVABILITY_SHUTDOWN_TIMEOUT_SECONDS`); a -queue that cannot drain before its deadline is abandoned with a report. -A hard kill loses whatever was still queued. Note the google stdout -format sets no `time` key: stdout emission is synchronous, so the -agent's receive time is the correct event time. - -Processes that fork or restore from memory snapshots do not preserve -threads or network clients. Call `restart_observability()` from the -post-restore or post-fork hook (for example gunicorn `post_fork` when -using `--preload`, or a Modal post-snapshot hook) — it closes and -rebuilds destinations from configuration, and must only be called from -single-threaded lifecycle moments, before serving traffic. It is a -no-op when observability is disabled, so the kill switch holds across -forks and restores. - -Request and operation logs include two timing views: - -- `timings_ms` and `timing_counts` are flat inclusive aggregates by segment - name, intended for quick scanning and compatibility with existing log - queries. -- `segment_tree` is an ordered nested view of segment occurrences. Repeated - sibling segments are preserved as separate entries, and safe scalar segment - attributes are included so callers can distinguish settings such as - `simulation_kind=baseline` versus `simulation_kind=reform`. - -Core structured log fields take precedence over caller-provided attributes with -the same keys. - -On runtimes that do not provide Application Default Credentials, set -`GCP_CREDENTIALS_JSON` to a service account JSON document. The Google Cloud -Logging destination will materialize it into a temporary credentials file and -pass those credentials directly to the Google client. If the credential -bootstrap fails, observability fails open and continues without raising into -application code. - -Prefer OIDC-based Workload Identity Federation over long-lived service account -keys when the runtime can provide an OIDC subject token. Modal injects -generated identity tokens into Function containers through -`MODAL_IDENTITY_TOKEN`; other runtimes can provide -`OBSERVABILITY_GOOGLE_OIDC_TOKEN`. The runtime needs these values: +Both adapters extract W3C trace context and +`X-PolicyEngine-Request-Id`, create one server span, emit one completion +record, record request metrics, and clear context-local state. Repeated calls +reuse the first runtime associated with the application. + +## Manual operations and spans + +Use an operation for one complete non-HTTP invocation. It emits one completion +record and records duration, count, and error metrics. Use spans for child +steps; child spans do not emit additional operation completion records. + +```python +with runtime.operation( + "simulation.run", + attributes={"backend": "modal"}, +): + with runtime.span("simulation.build"): + simulation = build_simulation() + result = calculate(simulation) +``` -```bash -OBSERVABILITY_GOOGLE_OIDC_TOKEN=OIDC_TOKEN_FROM_RUNTIME -OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER=projects/PROJECT_NUMBER/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID -OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL=observability-writer@PROJECT_ID.iam.gserviceaccount.com -OBSERVABILITY_GOOGLE_CLOUD_PROJECT=PROJECT_ID +Operations and spans support synchronous and asynchronous context management +and decoration: + +```python +@runtime.span("simulation.calculate") +async def calculate(simulation): + return await simulation.calculate() +``` + +Function arguments and return values are never captured. Telemetry failures do +not change the function return value or replace an application exception. + +## Events, application logs, and handled exceptions + +The runtime accepts only configured attribute keys and scalar values. + +```python +runtime.set_context(auth_result="accepted", simulation_id="sim-123") +runtime.event("simulation.dispatched", attributes={"backend": "modal"}) + +try: + load_result() +except ValueError as error: + runtime.record_exception(error, handled=True) +``` + +To route standard-library logs through the versioned JSON schema, instrument a +specific logger or configure root capture explicitly: + +```python +import logging +from policyengine_observability import instrument_logging + +logger = logging.getLogger("policyengine.api") +instrument_logging(logger, runtime) +logger.info( + "Simulation accepted", + extra={"policyengine_attributes": {"backend": "modal"}}, +) +``` + +The handler ignores Google, gRPC, OTel, and this package's own loggers to avoid +recursive export failures. + +## Outbound HTTP and asynchronous dispatch + +Only the supplied HTTPX client is modified: + +```python +import httpx +from policyengine_observability import instrument_httpx + +client = httpx.AsyncClient() +instrument_httpx(client, runtime) +``` + +The request hook injects active W3C context and the PolicyEngine request ID. +Other clients in the process remain unchanged. + +For a queued simulation, serialize the bounded correlation context with the +job request: + +```python +request.observability_context = runtime.capture_context() +``` + +Then restore it around the worker invocation: + +```python +with runtime.operation( + "simulation.run", + remote_context=request.observability_context, +): + return run_simulation(request) +``` + +A direct continuation that starts within five minutes uses the dispatched span +as its parent. An older invocation, independent retry, or aggregate operation +starts a new trace with a link to the dispatch span. + +## Modal logging and snapshots + +Modal writes the same JSON immediately to standard output and may also send a +copy through one bounded background Cloud Logging writer per process: + +```python +from policyengine_observability import ( + GoogleCloudLoggingConfig, + LoggingConfig, +) + +config = ObservabilityConfig.from_env( + service=service, + deployment=DeploymentIdentity( + environment="production", + platform="modal", + ), + google_cloud_project_id="policyengine-observability", + logging=LoggingConfig( + remote=GoogleCloudLoggingConfig( + project_id="policyengine-observability", + log_name="policyengine-api-v1-modal", + queue_capacity=1_000, + batch_size=100, + write_timeout_seconds=5, + ) + ), +) +runtime = configure(config) +``` + +Remote logging starts no client until the background worker receives its first +record. The calling thread uses `put_nowait`; a full queue drops the newest +record and records a local counter. + +After a Modal memory snapshot restores, rebuild process-local queues, threads, +credentials, and exporters before accepting work: + +```python +@modal.enter(snap=False) +def restore_process_state(self): + self.runtime.restart_after_snapshot() ``` -When `MODAL_IDENTITY_TOKEN` or `OBSERVABILITY_GOOGLE_OIDC_TOKEN` is present -alongside `OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER`, the Google Cloud -Logging destination writes a temporary external-account credential -configuration and passes those credentials directly to the Cloud Logging -client. If `OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL` is present, the -configuration uses service account impersonation. This keeps observability -credentials separate from any application-level `GOOGLE_APPLICATION_CREDENTIALS` -or `GCP_CREDENTIALS_JSON` used by the service for other Google clients. +## Failure behavior and shutdown -The Google Cloud setup needs: +Remote telemetry delivery is best effort. Exporters have bounded queues, +batches, retries, network deadlines, and shutdown periods. Missing credentials, +invalid configuration, unavailable DNS, denied permissions, collector failure, +queue saturation, and shutdown timeouts produce rate-limited JSON diagnostics +on standard error. Those diagnostics do not enter the failing exporter. -- A Workload Identity Pool and OIDC provider whose issuer matches Modal's OIDC - issuer, `https://oidc.modal.com`. -- Attribute mapping for the Modal token claims you want to authorize, such as - `google.subject=assertion.sub`. -- A service account with `roles/logging.logWriter` on the log project. -- An IAM binding granting the workload identity principal - `roles/iam.workloadIdentityUser` on that service account. +Call `runtime.shutdown()` during orderly process shutdown. The call is bounded +and safe to repeat. -For the fixed PolicyEngine Google Cloud destination, see -[`docs/operations/google-cloud-stage3-runbook.md`](docs/operations/google-cloud-stage3-runbook.md). +The operating policy, workload allowlist, Google Cloud deployment plan, and +rollback procedure are in +[`docs/operations/api-v1-observability.md`](docs/operations/api-v1-observability.md) +and [`deploy/gcp/README.md`](deploy/gcp/README.md). ## Release workflow -Changes should include a Towncrier fragment in `changelog.d/`. Pull requests -run changelog, Ruff, and coverage checks. Pushes to `main` run the same gates, -then publish a versioning commit that builds the changelog and bumps -`pyproject.toml`. That versioning commit publishes the package to PyPI through -trusted publishing, creates a matching git tag, and opens a GitHub release. +Changes include a Towncrier fragment in `changelog.d/`. Pull requests run Ruff, +tests, and coverage checks. The release workflow builds distributions and +publishes through PyPI trusted publishing. ## License -Code in this repository is released under the [MIT License](LICENSE). Original text and figures are released under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) with attribution to PolicyEngine. Third-party data and materials keep their own terms. +Code in this repository is released under the [MIT License](LICENSE). Original +text and figures are released under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) +with attribution to PolicyEngine. Third-party materials retain their terms. diff --git a/changelog.d/29.breaking.md b/changelog.d/29.breaking.md new file mode 100644 index 0000000..11ae2ab --- /dev/null +++ b/changelog.d/29.breaking.md @@ -0,0 +1,6 @@ +Replace the implicit global runtime and version 1 decorators with an explicit +runtime returned by `configure`. Version 2 adds runtime-bound operations and +spans, idempotent Flask and FastAPI request adapters, explicit HTTPX and Python +logging instrumentation, bounded Modal Cloud Logging delivery, authenticated +OTLP export, and allowlisted cross-service correlation. Consumers must replace +removed version 1 imports before upgrading. diff --git a/deploy/gcp/README.md b/deploy/gcp/README.md new file mode 100644 index 0000000..a720485 --- /dev/null +++ b/deploy/gcp/README.md @@ -0,0 +1,181 @@ +# Google Cloud deployment plan + +This directory defines the centralized API v1 observability resources in the +`policyengine-observability` project. The workload boundary is defined in +[`workload-inventory.yaml`](workload-inventory.yaml). Applications absent from +that inventory receive no credentials or destination permissions. + +These files are a reviewable deployment plan. Applying them changes live IAM, +Cloud Logging routing, Cloud Run, and monitoring resources and therefore +requires an operator-approved deployment window. + +## Resources + +| File | Resource | +| --- | --- | +| `collector/config.yaml` | OTLP gRPC receiver, bounded processors, and Google Telemetry API exporter for traces and metrics | +| `collector/Dockerfile` | Google-built OTel Collector 0.160.0 plus the reviewed configuration | +| `collector/service.yaml` | Authenticated Cloud Run collector with fixed CPU, memory, concurrency, health checks, and scaling bounds | +| `log-routing.yaml` | Exact Cloud Run source sinks, restricted Modal direct-log sink, and `_Default` duplicate exclusion | +| `iam.yaml` | Collector and Modal service accounts, Cloud Run invokers, and a dedicated Modal API v1 identity provider | +| `dashboard.json` | Initial request, latency, error, dropped-item, and exporter-failure dashboard | +| `alerts.yaml` | Initial alert policy inputs | +| `verify.sh` | Read-only resource and routing checks after deployment | + +The collector accepts traces and metrics. Application logs do not enter the +collector. Cloud Run JSON output uses source-project sinks, while authorized +Modal processes use the package's bounded Cloud Logging writer. + +The initial collector tail policy retains 100% of traces. Separate error and +30-second latency policies are evaluated before the general policy. If the +general percentage is reduced after the volume review, those two policies keep +error and slow traces. SDK head sampling must remain at 100% for the collector +to receive spans needed for this decision. + +## Deployment order + +### 1. Enable services + +```bash +gcloud services enable \ + artifactregistry.googleapis.com \ + cloudbuild.googleapis.com \ + cloudresourcemanager.googleapis.com \ + iamcredentials.googleapis.com \ + logging.googleapis.com \ + monitoring.googleapis.com \ + run.googleapis.com \ + sts.googleapis.com \ + telemetry.googleapis.com \ + tracing.googleapis.com \ + --project=policyengine-observability +``` + +### 2. Create identities + +Create these service accounts in `policyengine-observability`: + +```text +policyengine-otel-collector@policyengine-observability.iam.gserviceaccount.com +policyengine-api-v1-modal@policyengine-observability.iam.gserviceaccount.com +``` + +Grant only the roles listed in [`iam.yaml`](iam.yaml). The collector receives +`roles/telemetry.writer` and `roles/serviceusage.serviceUsageConsumer`. The Modal identity receives +`roles/logging.logWriter` and collector invocation permission. The four +existing Cloud Run identities in the inventory receive collector invocation +permission on the collector service only. + +Create a separate `modal-api-v1` workload identity pool and provider using the +issuer, audience, mappings, workspace, environment, and application condition +in `iam.yaml`. Do not modify the existing `modal/modal` provider during this +deployment; it belongs to applications excluded from this change. Grant the +new provider permission to impersonate only the API v1 Modal service account. + +Before enabling the provider, decode one production and one staging Modal +identity token locally and confirm that `workspace_id`, `environment_name`, +and `app_name` exactly match the reviewed condition. + +### 3. Build and deploy the collector + +```bash +gcloud artifacts repositories create observability \ + --repository-format=docker \ + --location=us-central1 \ + --immutable-tags \ + --project=policyengine-observability + +gcloud builds submit deploy/gcp/collector \ + --tag=us-central1-docker.pkg.dev/policyengine-observability/observability/otel-collector:0.160.0-api-v1-1 \ + --project=policyengine-observability + +gcloud run services replace deploy/gcp/collector/service.yaml \ + --region=us-central1 \ + --project=policyengine-observability +``` + +Apply `roles/run.invoker` bindings for the five identities listed in +`iam.yaml`. Do not grant unauthenticated invocation. Record the HTTPS service +URL as both `OTEL_EXPORTER_OTLP_ENDPOINT` and +`POLICYENGINE_OTEL_GOOGLE_AUDIENCE` in participating service configuration. + +### 4. Configure log routing + +Create one aggregated sink in each source project using the exact service and +schema filters in [`log-routing.yaml`](log-routing.yaml). Grant each generated +sink writer identity `roles/logging.bucketWriter` on the central +`policyengine-observability` bucket. + +Update `policyengine-observability-app-logs` to the listed direct-log filter. +Add the listed exclusion to `_Default`; this prevents a direct Modal log from +being stored in both `_Default` and the analytics bucket. Preserve the existing +Cloud Audit Log exclusions. + +After routing one synthetic record per participating service, confirm each +`insertId` exists exactly once in the central project. + +### 5. Create dashboard and alerts + +```bash +gcloud monitoring dashboards create \ + --config-from-file=deploy/gcp/dashboard.json \ + --project=policyengine-observability +``` + +Create the API-ready alert policies with: + +```bash +.venv/bin/python deploy/gcp/create_alerts.py +``` + +The script is idempotent by policy display name. It leaves notification-channel +configuration empty when the project has no channel; add operator-owned channel +identifiers after creating the relevant email, Slack, or paging destination. + +### 6. Verify before consumer deployment + +```bash +bash deploy/gcp/verify.sh +``` + +Then use an approved workload identity to send one trace and metric. Attempt +the same request with a synthetic Modal token whose application name is not in +the inventory; token exchange or collector invocation must return permission +denial. Do not invoke an excluded application to perform this check. + +For an operator-run Cloud Run identity check, temporarily grant the operator +`roles/iam.serviceAccountTokenCreator` on one inventoried runtime identity, run: + +```bash +.venv/bin/python deploy/gcp/verify_otel.py \ + --endpoint=https://policyengine-api-v1-otel-collector-790230211054.us-central1.run.app \ + --service-account=sim-entry-beta-runtime@policyengine-simulation-entry.iam.gserviceaccount.com +``` + +Remove the temporary operator binding immediately after the check. The script +requires an authenticated `gcloud` session, sends one trace and metric, verifies +both Google Cloud stores, and confirms that the collector rejects OTLP logs. + +Use [`verify_modal_wif.py`](verify_modal_wif.py) with the Modal CLI to run an +allowed app name and a synthetic denied app name. The remote function exchanges +its automatically injected Modal OIDC token, verifies service-account access, +invokes the collector, and writes one routing record without exposing any +token. Run an allowed app name from a temporary non-allowlisted environment to +verify the environment restriction, then delete that environment. + +## Rollback + +1. Remove the OTel endpoint from participating service configuration. Local + structured logging continues and no remote OTel exporter is created. +2. Remove Modal remote logging configuration. Modal JSON output continues. +3. Revert each participating service to its previous package version and + deployment revision. +4. Remove the new source sinks and restore the prior central direct-log sink + filter and `_Default` exclusion state. +5. Remove invoker bindings, disable the `modal-api-v1` provider, and disable or + delete the collector service. +6. Keep the central bucket during the retention period unless the stored data + itself caused the incident. + +Rollback does not modify the existing `modal/modal` provider or any excluded +application deployment. diff --git a/deploy/gcp/alerts.yaml b/deploy/gcp/alerts.yaml new file mode 100644 index 0000000..cfd617e --- /dev/null +++ b/deploy/gcp/alerts.yaml @@ -0,0 +1,42 @@ +policies: + - display_name: API v1 exporter failures + condition_type: prometheus_query_language + query: 'sum(increase({"policyengine.telemetry.exporter.failure"}[5m])) > 0' + duration: 0s + evaluation_interval: 60s + disable_metric_validation: true + - display_name: API v1 dropped telemetry + condition_type: prometheus_query_language + query: 'sum(increase({"policyengine.telemetry.dropped"}[5m])) > 0' + duration: 0s + evaluation_interval: 60s + disable_metric_validation: true + - display_name: API v1 elevated error rate + condition_type: prometheus_query_language + query: 'sum(increase({"policyengine.error.count"}[10m])) > 1' + duration: 0s + evaluation_interval: 60s + disable_metric_validation: true + - display_name: API v1 high request latency + condition_type: prometheus_query_language + query: >- + histogram_quantile(0.99, sum by (le) + (rate({"policyengine.request.duration_bucket"}[5m]))) > 30 + duration: 300s + evaluation_interval: 60s + disable_metric_validation: true + - display_name: API v1 collector unavailable + condition_type: metric_threshold + metric_type: run.googleapis.com/service_health_count + resource_label_service_name: policyengine-api-v1-otel-collector + metric_label_service_health: UNHEALTHY + comparison: COMPARISON_GT + threshold: 0 + duration: 300s + - display_name: API v1 monthly log ingestion above 10 GiB + condition_type: metric_threshold + metric_type: logging.googleapis.com/billing/log_bucket_monthly_bytes_ingested + metric_label_log_bucket_id: policyengine-observability + comparison: COMPARISON_GT + threshold: 10737418240 + duration: 0s diff --git a/deploy/gcp/collector/Dockerfile b/deploy/gcp/collector/Dockerfile new file mode 100644 index 0000000..ae3f632 --- /dev/null +++ b/deploy/gcp/collector/Dockerfile @@ -0,0 +1,6 @@ +FROM us-docker.pkg.dev/cloud-ops-agents-artifacts/google-cloud-opentelemetry-collector/otelcol-google:0.160.0 + +COPY config.yaml /etc/otelcol-google/config.yaml + +ENTRYPOINT ["/otelcol-google"] +CMD ["--config=/etc/otelcol-google/config.yaml"] diff --git a/deploy/gcp/collector/config.yaml b/deploy/gcp/collector/config.yaml new file mode 100644 index 0000000..b9adbb5 --- /dev/null +++ b/deploy/gcp/collector/config.yaml @@ -0,0 +1,71 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:8080 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 384 + spike_limit_mib: 64 + resource/destination: + attributes: + - key: gcp.project_id + value: policyengine-observability + action: upsert + tail_sampling: + decision_wait: 30s + num_traces: 50000 + expected_new_traces_per_sec: 500 + policies: + - name: retain-errors + type: status_code + status_code: + status_codes: [ERROR] + - name: retain-slow-operations + type: latency + latency: + threshold_ms: 30000 + - name: initial-full-sample + type: probabilistic + probabilistic: + sampling_percentage: 100 + batch: + send_batch_size: 200 + send_batch_max_size: 1000 + timeout: 5s + +exporters: + otlp_grpc: + endpoint: telemetry.googleapis.com:443 + auth: + authenticator: googleclientauth + sending_queue: + enabled: true + queue_size: 2000 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + +extensions: + googleclientauth: + health_check: + endpoint: 0.0.0.0:13133 + +service: + extensions: [googleclientauth, health_check] + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, resource/destination, tail_sampling, batch] + exporters: [otlp_grpc] + metrics: + receivers: [otlp] + processors: [memory_limiter, resource/destination, batch] + exporters: [otlp_grpc] + telemetry: + logs: + level: info diff --git a/deploy/gcp/collector/service.yaml b/deploy/gcp/collector/service.yaml new file mode 100644 index 0000000..5b5316f --- /dev/null +++ b/deploy/gcp/collector/service.yaml @@ -0,0 +1,45 @@ +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: policyengine-api-v1-otel-collector + namespace: "790230211054" + labels: + cloud.googleapis.com/location: us-central1 + annotations: + run.googleapis.com/ingress: all +spec: + template: + metadata: + annotations: + autoscaling.knative.dev/minScale: "1" + autoscaling.knative.dev/maxScale: "10" + run.googleapis.com/cpu-throttling: "false" + run.googleapis.com/startup-cpu-boost: "true" + spec: + containerConcurrency: 100 + timeoutSeconds: 30 + serviceAccountName: policyengine-otel-collector@policyengine-observability.iam.gserviceaccount.com + containers: + - name: collector + image: us-central1-docker.pkg.dev/policyengine-observability/observability/otel-collector:0.160.0-api-v1-1 + ports: + - name: h2c + containerPort: 8080 + resources: + limits: + cpu: "1" + memory: 512Mi + startupProbe: + httpGet: + path: / + port: 13133 + periodSeconds: 2 + timeoutSeconds: 1 + failureThreshold: 30 + livenessProbe: + httpGet: + path: / + port: 13133 + periodSeconds: 30 + timeoutSeconds: 2 + failureThreshold: 3 diff --git a/deploy/gcp/create_alerts.py b/deploy/gcp/create_alerts.py new file mode 100644 index 0000000..025e98a --- /dev/null +++ b/deploy/gcp/create_alerts.py @@ -0,0 +1,250 @@ +"""Create the initial API v1 Cloud Monitoring alert policies idempotently.""" + +from __future__ import annotations + +import json +import subprocess +import urllib.parse +import urllib.request +from typing import Any + +PROJECT = "policyengine-observability" +API_ROOT = f"https://monitoring.googleapis.com/v3/projects/{PROJECT}" + + +def _threshold_condition( + *, + display_name: str, + filter_value: str, + comparison: str, + threshold: float, + duration: str, + alignment_period: str, + aligner: str, + reducer: str, +) -> dict[str, Any]: + return { + "displayName": display_name, + "conditionThreshold": { + "filter": filter_value, + "comparison": comparison, + "thresholdValue": threshold, + "duration": duration, + "aggregations": [ + { + "alignmentPeriod": alignment_period, + "perSeriesAligner": aligner, + "crossSeriesReducer": reducer, + } + ], + "trigger": {"count": 1}, + }, + } + + +def _promql_condition( + *, + display_name: str, + query: str, + duration: str, + alert_rule: str, +) -> dict[str, Any]: + return { + "displayName": display_name, + "conditionPrometheusQueryLanguage": { + "query": query, + "duration": duration, + "evaluationInterval": "60s", + "alertRule": alert_rule, + "ruleGroup": "policyengine_api_v1", + "disableMetricValidation": True, + }, + } + + +def _policy( + display_name: str, + condition: dict[str, Any], + documentation: str, +) -> dict[str, Any]: + return { + "displayName": display_name, + "combiner": "OR", + "enabled": True, + "notificationChannels": [], + "documentation": { + "content": documentation, + "mimeType": "text/markdown", + }, + "alertStrategy": {"autoClose": "1800s"}, + "conditions": [condition], + } + + +POLICIES = [ + _policy( + "API v1 exporter failures", + _promql_condition( + display_name="Exporter failure counter increased", + query=( + 'sum(increase({"policyengine.telemetry.exporter.failure"}' + "[5m])) > 0" + ), + duration="0s", + alert_rule="ExporterFailure", + ), + "The API v1 runtime reported at least one telemetry exporter failure.", + ), + _policy( + "API v1 dropped telemetry", + _promql_condition( + display_name="Dropped telemetry counter increased", + query=( + 'sum(increase({"policyengine.telemetry.dropped"}[5m])) > 0' + ), + duration="0s", + alert_rule="DroppedTelemetry", + ), + "An API v1 bounded telemetry queue dropped at least one item.", + ), + _policy( + "API v1 elevated error rate", + _promql_condition( + display_name="More than one application error in ten minutes", + query='sum(increase({"policyengine.error.count"}[10m])) > 1', + duration="0s", + alert_rule="ElevatedErrorRate", + ), + "API v1 application error counters increased more than once in ten minutes.", + ), + _policy( + "API v1 high request latency", + _promql_condition( + display_name="P99 request duration above 30 seconds", + query=( + "histogram_quantile(0.99, sum by (le) " + '(rate({"policyengine.request.duration_bucket"}[5m]))) > 30' + ), + duration="300s", + alert_rule="HighRequestLatency", + ), + "API v1 P99 request duration exceeded 30 seconds for five minutes.", + ), + _policy( + "API v1 collector unavailable", + _threshold_condition( + display_name="Collector Cloud Run service is unhealthy", + filter_value=( + 'resource.type="cloud_run_revision" AND ' + 'metric.type="run.googleapis.com/service_health_count" AND ' + 'resource.label."service_name"=' + '"policyengine-api-v1-otel-collector" AND ' + 'metric.label."service_health"="UNHEALTHY"' + ), + comparison="COMPARISON_GT", + threshold=0, + duration="300s", + alignment_period="60s", + aligner="ALIGN_MEAN", + reducer="REDUCE_MAX", + ), + "The authenticated API v1 collector reported an unhealthy revision.", + ), + _policy( + "API v1 monthly log ingestion above 10 GiB", + _threshold_condition( + display_name="Central analytics bucket exceeds 10 GiB this month", + filter_value=( + 'resource.type="global" AND ' + 'metric.type="logging.googleapis.com/billing/' + 'log_bucket_monthly_bytes_ingested" AND ' + 'metric.label."log_bucket_id"="policyengine-observability"' + ), + comparison="COMPARISON_GT", + threshold=10_737_418_240, + duration="0s", + alignment_period="1800s", + aligner="ALIGN_MAX", + reducer="REDUCE_SUM", + ), + "The central API v1 analytics bucket exceeded 10 GiB of month-to-date ingestion.", + ), +] + + +def _token() -> str: + return subprocess.check_output( + ["gcloud", "auth", "print-access-token"], + text=True, + ).strip() + + +def _request( + url: str, + *, + token: str, + payload: dict[str, Any] | None = None, +) -> dict[str, Any]: + request = urllib.request.Request( + url, + data=json.dumps(payload).encode() if payload is not None else None, + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + method="POST" if payload is not None else "GET", + ) + with urllib.request.urlopen(request, timeout=30) as response: + return json.loads(response.read()) + + +def main() -> None: + token = _token() + existing: dict[str, str] = {} + page_token = "" + while True: + query = ( + urllib.parse.urlencode({"pageToken": page_token}) + if page_token + else "" + ) + suffix = f"?{query}" if query else "" + response = _request( + f"{API_ROOT}/alertPolicies{suffix}", + token=token, + ) + for policy in response.get("alertPolicies", []): + existing[str(policy["displayName"])] = str(policy["name"]) + page_token = str(response.get("nextPageToken", "")) + if not page_token: + break + + results = [] + for policy in POLICIES: + display_name = str(policy["displayName"]) + if display_name in existing: + results.append( + { + "displayName": display_name, + "name": existing[display_name], + "status": "existing", + } + ) + continue + created = _request( + f"{API_ROOT}/alertPolicies", + token=token, + payload=policy, + ) + results.append( + { + "displayName": display_name, + "name": created["name"], + "status": "created", + } + ) + print(json.dumps(results, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/deploy/gcp/dashboard.json b/deploy/gcp/dashboard.json new file mode 100644 index 0000000..e525517 --- /dev/null +++ b/deploy/gcp/dashboard.json @@ -0,0 +1,183 @@ +{ + "displayName": "PolicyEngine API v1 observability", + "mosaicLayout": { + "columns": 12, + "tiles": [ + { + "xPos": 0, + "yPos": 0, + "width": 6, + "height": 4, + "widget": { + "title": "Request rate by service", + "xyChart": { + "dataSets": [{ + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"prometheus.googleapis.com/policyengine.request.count/counter\"", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_RATE", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": ["resource.label.\"job\""] + } + } + }, + "plotType": "LINE" + }] + } + } + }, + { + "xPos": 6, + "yPos": 0, + "width": 6, + "height": 4, + "widget": { + "title": "Request duration", + "xyChart": { + "dataSets": [{ + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"prometheus.googleapis.com/policyengine.request.duration/histogram\"", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_PERCENTILE_99" + } + } + }, + "plotType": "LINE" + }] + } + } + }, + { + "xPos": 0, + "yPos": 4, + "width": 6, + "height": 4, + "widget": { + "title": "Application errors", + "xyChart": { + "dataSets": [{ + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"prometheus.googleapis.com/policyengine.error.count/counter\"", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_RATE", + "crossSeriesReducer": "REDUCE_SUM" + } + } + }, + "plotType": "STACKED_AREA" + }] + } + } + }, + { + "xPos": 6, + "yPos": 4, + "width": 6, + "height": 4, + "widget": { + "title": "Dropped telemetry and exporter failures", + "xyChart": { + "dataSets": [ + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"prometheus.googleapis.com/policyengine.telemetry.dropped/counter\"", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_RATE", + "crossSeriesReducer": "REDUCE_SUM" + } + } + }, + "plotType": "LINE" + }, + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "metric.type=\"prometheus.googleapis.com/policyengine.telemetry.exporter.failure/counter\"", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_RATE", + "crossSeriesReducer": "REDUCE_SUM" + } + } + }, + "plotType": "LINE" + } + ] + } + } + }, + { + "xPos": 0, + "yPos": 8, + "width": 6, + "height": 4, + "widget": { + "title": "Collector health", + "xyChart": { + "dataSets": [{ + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "resource.type=\"cloud_run_revision\" AND metric.type=\"run.googleapis.com/service_health_count\" AND resource.label.\"service_name\"=\"policyengine-api-v1-otel-collector\"", + "aggregation": { + "alignmentPeriod": "60s", + "perSeriesAligner": "ALIGN_MEAN", + "crossSeriesReducer": "REDUCE_SUM", + "groupByFields": ["metric.label.\"service_health\""] + } + } + }, + "plotType": "LINE" + }] + } + } + }, + { + "xPos": 6, + "yPos": 8, + "width": 6, + "height": 4, + "widget": { + "title": "Telemetry ingestion volume", + "xyChart": { + "dataSets": [ + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "resource.type=\"global\" AND metric.type=\"logging.googleapis.com/billing/log_bucket_bytes_ingested\" AND metric.label.\"log_bucket_id\"=\"policyengine-observability\"", + "aggregation": { + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_SUM", + "crossSeriesReducer": "REDUCE_SUM" + } + } + }, + "plotType": "LINE" + }, + { + "timeSeriesQuery": { + "timeSeriesFilter": { + "filter": "resource.type=\"global\" AND metric.type=\"monitoring.googleapis.com/billing/samples_ingested\" AND metric.label.\"metric_domain\"=\"prometheus.googleapis.com\"", + "aggregation": { + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_SUM", + "crossSeriesReducer": "REDUCE_SUM" + } + } + }, + "plotType": "LINE" + } + ] + } + } + } + ] + } +} diff --git a/deploy/gcp/iam.yaml b/deploy/gcp/iam.yaml new file mode 100644 index 0000000..87e06ac --- /dev/null +++ b/deploy/gcp/iam.yaml @@ -0,0 +1,36 @@ +service_accounts: + collector: + email: policyengine-otel-collector@policyengine-observability.iam.gserviceaccount.com + project_roles: + - roles/telemetry.writer + - roles/serviceusage.serviceUsageConsumer + modal: + email: policyengine-api-v1-modal@policyengine-observability.iam.gserviceaccount.com + project_roles: + - roles/logging.logWriter + collector_roles: + - roles/run.invoker + +collector_invokers: + - policyengine-api-cr-runtime@policyengine-api.iam.gserviceaccount.com + - policyengine-api-cr-staging@policyengine-api.iam.gserviceaccount.com + - sim-entry-prod-runtime@policyengine-simulation-entry.iam.gserviceaccount.com + - sim-entry-beta-runtime@policyengine-simulation-entry.iam.gserviceaccount.com + - policyengine-api-v1-modal@policyengine-observability.iam.gserviceaccount.com + +modal_workload_identity: + pool_id: modal-api-v1 + provider_id: modal-api-v1 + issuer: https://oidc.modal.com + allowed_audience: oidc.modal.com + workspace_id: ac-AVO94gmhUEaFCuduiKdpRJ + attribute_mapping: + google.subject: assertion.app_name + attribute.app_name: assertion.app_name + attribute.environment_name: assertion.environment_name + attribute.workspace_id: assertion.workspace_id + attribute_condition: >- + assertion.workspace_id == "ac-AVO94gmhUEaFCuduiKdpRJ" && + (assertion.environment_name == "main" || assertion.environment_name == "staging") && + (assertion.app_name == "policyengine-simulation-gateway" || + assertion.app_name.matches("^policyengine-simulation-py[0-9]+-[0-9]+-[0-9]+$")) diff --git a/deploy/gcp/log-routing.yaml b/deploy/gcp/log-routing.yaml new file mode 100644 index 0000000..2e9a7b4 --- /dev/null +++ b/deploy/gcp/log-routing.yaml @@ -0,0 +1,37 @@ +destination: + project: policyengine-observability + location: global + bucket: policyengine-observability + direct_log_name: policyengine-api-v1-modal + +source_sinks: + - project: policyengine-api + name: api-v1-central-observability + filter: >- + resource.type="cloud_run_revision" AND + (resource.labels.service_name="policyengine-api" OR + resource.labels.service_name="policyengine-api-staging") AND + jsonPayload.schema_version="policyengine.observability.v2" + - project: policyengine-simulation-entry + name: api-v1-central-observability + filter: >- + resource.type="cloud_run_revision" AND + (resource.labels.service_name="policyengine-simulation-entry" OR + resource.labels.service_name="policyengine-simulation-entry-staging") AND + jsonPayload.schema_version="policyengine.observability.v2" + +central_direct_sink: + name: policyengine-observability-app-logs + filter: >- + LOG_ID("policyengine-api-v1-modal") AND + jsonPayload.schema_version="policyengine.observability.v2" AND + jsonPayload."service.namespace"="policyengine.api-v1" + +default_bucket_exclusion: + name: exclude-api-v1-direct-observability + filter: >- + LOG_ID("policyengine-api-v1-modal") AND + jsonPayload.schema_version="policyengine.observability.v2" AND + jsonPayload."service.namespace"="policyengine.api-v1" AND + (jsonPayload."service.name"="policyengine-simulation-gateway" OR + jsonPayload."service.name"=~"^policyengine-simulation-py[0-9]+-[0-9]+-[0-9]+$") diff --git a/deploy/gcp/verify.sh b/deploy/gcp/verify.sh new file mode 100644 index 0000000..e454dfc --- /dev/null +++ b/deploy/gcp/verify.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +project="policyengine-observability" +region="us-central1" +collector="policyengine-api-v1-otel-collector" + +gcloud logging buckets describe policyengine-observability \ + --location=global \ + --project="${project}" \ + --format='value(name,retentionDays,analyticsEnabled)' + +gcloud run services describe "${collector}" \ + --region="${region}" \ + --project="${project}" \ + --format='value(status.url,spec.template.spec.serviceAccountName)' + +gcloud projects get-iam-policy "${project}" \ + --flatten='bindings[].members' \ + --filter='bindings.role:roles/telemetry.writer OR bindings.role:roles/serviceusage.serviceUsageConsumer OR bindings.role:roles/logging.logWriter' \ + --format='table(bindings.role,bindings.members)' + +gcloud logging views get-iam-policy _AllLogs \ + --bucket=policyengine-observability \ + --location=global \ + --project="${project}" \ + --format=json + +gcloud iam workload-identity-pools providers describe modal-api-v1 \ + --workload-identity-pool=modal-api-v1 \ + --location=global \ + --project="${project}" \ + --format='yaml(state,attributeCondition,attributeMapping,oidc)' + +for source_project in policyengine-api policyengine-simulation-entry; do + gcloud logging sinks describe api-v1-central-observability \ + --project="${source_project}" \ + --format='yaml(destination,filter,writerIdentity)' +done + +gcloud logging sinks describe policyengine-observability-app-logs \ + --project="${project}" \ + --format='yaml(destination,filter)' + +gcloud logging sinks describe _Default \ + --project="${project}" \ + --format='yaml(filter,exclusions)' diff --git a/deploy/gcp/verify_modal_wif.py b/deploy/gcp/verify_modal_wif.py new file mode 100644 index 0000000..80aab36 --- /dev/null +++ b/deploy/gcp/verify_modal_wif.py @@ -0,0 +1,194 @@ +"""Verify allowed and denied Modal identities against Google WIF. + +Set ``POLICYENGINE_WIF_TEST_APP`` in the local process before ``modal run``. +The script never returns or prints the Modal identity token or Google tokens. +""" + +# pyright: reportMissingImports=false + +from __future__ import annotations + +import base64 +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request + +import modal + +PROJECT = "policyengine-observability" +PROJECT_NUMBER = "790230211054" +POOL = "modal-api-v1" +PROVIDER = "modal-api-v1" +SERVICE_ACCOUNT = "policyengine-api-v1-modal@policyengine-observability.iam.gserviceaccount.com" +COLLECTOR = "https://policyengine-api-v1-otel-collector-790230211054.us-central1.run.app" +APP_NAME = os.environ.get( + "POLICYENGINE_WIF_TEST_APP", + "policyengine-observability-wif-denied-test", +) + +app = modal.App(APP_NAME) + + +def _jwt_claims(token: str) -> dict[str, object]: + payload = token.split(".")[1] + payload += "=" * (-len(payload) % 4) + return json.loads(base64.urlsafe_b64decode(payload)) + + +def _post_form(url: str, values: dict[str, str]) -> dict[str, object]: + request = urllib.request.Request( + url, + data=urllib.parse.urlencode(values).encode(), + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + with urllib.request.urlopen(request, timeout=15) as response: + return json.loads(response.read()) + + +def _post_json( + url: str, + payload: dict[str, object], + *, + bearer_token: str, +) -> tuple[int, dict[str, object]]: + request = urllib.request.Request( + url, + data=json.dumps(payload).encode(), + headers={ + "Authorization": f"Bearer {bearer_token}", + "Content-Type": "application/json", + }, + ) + with urllib.request.urlopen(request, timeout=15) as response: + body = response.read() + return response.status, json.loads(body) if body else {} + + +def _error_result(error: urllib.error.HTTPError) -> dict[str, object]: + try: + payload = json.loads(error.read()) + except (json.JSONDecodeError, UnicodeDecodeError): + payload = {} + error_payload = payload.get("error", {}) + if isinstance(error_payload, dict): + error_name = error_payload.get("status") or error_payload.get("error") + else: + error_name = error_payload + return { + "http_status": error.code, + "error": error_name, + } + + +@app.function(timeout=60) +def verify_identity() -> dict[str, object]: + identity_token = os.environ["MODAL_IDENTITY_TOKEN"] + claims = _jwt_claims(identity_token) + safe_claims = { + key: claims.get(key) + for key in ( + "workspace_id", + "environment_name", + "app_name", + "function_name", + "aud", + "iss", + ) + } + audience = ( + "//iam.googleapis.com/projects/" + f"{PROJECT_NUMBER}/locations/global/workloadIdentityPools/{POOL}" + f"/providers/{PROVIDER}" + ) + try: + sts_payload = _post_form( + "https://sts.googleapis.com/v1/token", + { + "audience": audience, + "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", + "requested_token_type": ( + "urn:ietf:params:oauth:token-type:access_token" + ), + "scope": "https://www.googleapis.com/auth/cloud-platform", + "subject_token": identity_token, + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + }, + ) + except urllib.error.HTTPError as error: + return { + "claims": safe_claims, + "token_exchange": _error_result(error), + } + + federated_token = str(sts_payload["access_token"]) + service_account_url = ( + "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/" + f"{SERVICE_ACCOUNT}" + ) + access_status, access_payload = _post_json( + f"{service_account_url}:generateAccessToken", + { + "scope": ["https://www.googleapis.com/auth/cloud-platform"], + "lifetime": "300s", + }, + bearer_token=federated_token, + ) + service_access_token = str(access_payload["accessToken"]) + + identity_status, identity_payload = _post_json( + f"{service_account_url}:generateIdToken", + {"audience": COLLECTOR, "includeEmail": True}, + bearer_token=federated_token, + ) + collector_identity_token = str(identity_payload["token"]) + collector_request = urllib.request.Request( + COLLECTOR, + headers={"Authorization": f"Bearer {collector_identity_token}"}, + ) + try: + with urllib.request.urlopen(collector_request, timeout=15) as response: + collector_status = response.status + except urllib.error.HTTPError as error: + collector_status = error.code + + verification_id = f"modal-wif-{int(time.time())}" + logging_status, _ = _post_json( + "https://logging.googleapis.com/v2/entries:write", + { + "logName": f"projects/{PROJECT}/logs/policyengine-api-v1-modal", + "resource": { + "type": "global", + "labels": {"project_id": PROJECT}, + }, + "entries": [ + { + "insertId": verification_id, + "jsonPayload": { + "schema_version": "policyengine.observability.v2", + "verification_id": verification_id, + "service.namespace": "policyengine.api-v1", + "service.name": safe_claims["app_name"], + "message": "Modal WIF verification", + }, + } + ], + }, + bearer_token=service_access_token, + ) + return { + "claims": safe_claims, + "token_exchange": {"http_status": 200}, + "service_account_access": {"http_status": access_status}, + "service_account_identity": {"http_status": identity_status}, + "collector_http_status": collector_status, + "logging_http_status": logging_status, + "verification_id": verification_id, + } + + +@app.local_entrypoint() +def main() -> None: + print(json.dumps(getattr(verify_identity, "remote")(), sort_keys=True)) diff --git a/deploy/gcp/verify_otel.py b/deploy/gcp/verify_otel.py new file mode 100644 index 0000000..3acde9b --- /dev/null +++ b/deploy/gcp/verify_otel.py @@ -0,0 +1,253 @@ +"""Send and verify synthetic OTLP signals through the Cloud Run collector.""" + +from __future__ import annotations + +import argparse +import json +import secrets +import subprocess +import time +import urllib.error +import urllib.parse +import urllib.request + +import grpc +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, +) +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, +) +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ( + ExportLogsServiceRequest, +) +from opentelemetry.proto.collector.logs.v1.logs_service_pb2_grpc import ( + LogsServiceStub, +) +from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue +from opentelemetry.proto.logs.v1.logs_pb2 import ( + LogRecord, + ResourceLogs, + ScopeLogs, +) +from opentelemetry.proto.resource.v1.resource_pb2 import ( + Resource as ProtoResource, +) +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor + +PROJECT = "policyengine-observability" +METRIC_NAME = "policyengine.verification.counter" +METRIC_TYPE = f"prometheus.googleapis.com/{METRIC_NAME}/counter" + + +def _gcloud_output(*arguments: str) -> str: + return subprocess.check_output( + ["gcloud", *arguments], + text=True, + ).strip() + + +def _identity_token(service_account: str, audience: str) -> str: + return _gcloud_output( + "auth", + "print-identity-token", + f"--impersonate-service-account={service_account}", + f"--audiences={audience}", + ) + + +def _access_token() -> str: + return _gcloud_output("auth", "print-access-token") + + +def _authorized_get(url: str, token: str) -> dict[str, object]: + request = urllib.request.Request( + url, + headers={"Authorization": f"Bearer {token}"}, + ) + with urllib.request.urlopen(request, timeout=15) as response: + return json.loads(response.read()) + + +def _send_signals(endpoint: str, service_account: str) -> dict[str, object]: + identity_token = _identity_token(service_account, endpoint) + headers = (("authorization", f"Bearer {identity_token}"),) + credentials = grpc.ssl_channel_credentials() + resource = Resource.create( + { + "service.name": "policyengine-observability-verification", + "service.namespace": "policyengine.api-v1", + "service.version": "2.0.0-verification", + "service.role": "verification", + "deployment.environment.name": "staging", + "cloud.platform": "gcp_cloud_run", + "cloud.region": "us-central1", + } + ) + + trace_exporter = OTLPSpanExporter( + endpoint=endpoint, + credentials=credentials, + headers=headers, + timeout=10, + ) + trace_provider = TracerProvider(resource=resource) + trace_provider.add_span_processor(SimpleSpanProcessor(trace_exporter)) + tracer = trace_provider.get_tracer( + "policyengine.observability.verification" + ) + with tracer.start_as_current_span( + "policyengine.observability.verify" + ) as span: + span.set_attribute("policyengine.verification", True) + trace_id = f"{span.get_span_context().trace_id:032x}" + trace_provider.force_flush(timeout_millis=15_000) + trace_provider.shutdown() + + metric_exporter = OTLPMetricExporter( + endpoint=endpoint, + credentials=credentials, + headers=headers, + timeout=10, + ) + metric_reader = PeriodicExportingMetricReader( + metric_exporter, + export_interval_millis=60_000, + export_timeout_millis=10_000, + ) + metric_provider = MeterProvider( + resource=resource, + metric_readers=[metric_reader], + ) + meter = metric_provider.get_meter( + "policyengine.observability.verification" + ) + counter = meter.create_counter(METRIC_NAME) + counter.add(1, {"outcome": "success"}) + metric_provider.shutdown() + + host = urllib.parse.urlparse(endpoint).netloc + channel = grpc.secure_channel(host, credentials) + logs_stub = LogsServiceStub(channel) + log_request = ExportLogsServiceRequest( + resource_logs=[ + ResourceLogs( + resource=ProtoResource( + attributes=[ + KeyValue( + key="service.name", + value=AnyValue( + string_value="policyengine-observability-verification" + ), + ) + ] + ), + scope_logs=[ + ScopeLogs( + log_records=[ + LogRecord( + time_unix_nano=time.time_ns(), + severity_text="INFO", + body=AnyValue( + string_value="collector log rejection verification" + ), + ) + ] + ) + ], + ) + ] + ) + log_status = "accepted" + try: + logs_stub.Export(log_request, timeout=10, metadata=headers) + except grpc.RpcError as error: + log_status = error.code().name + finally: + channel.close() + + return { + "trace_id": trace_id, + "metric_type": METRIC_TYPE, + "log_export_status": log_status, + } + + +def _wait_for_storage( + *, trace_id: str, metric_type: str, timeout_seconds: int +) -> dict[str, object]: + access_token = _access_token() + deadline = time.monotonic() + timeout_seconds + start = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 300)) + end = time.strftime( + "%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() + timeout_seconds + 60) + ) + metric_filter = urllib.parse.quote(f'metric.type = "{metric_type}"') + metric_url = ( + f"https://monitoring.googleapis.com/v3/projects/{PROJECT}/timeSeries" + f"?filter={metric_filter}&interval.startTime={start}" + f"&interval.endTime={end}&view=HEADERS" + ) + trace_url = f"https://cloudtrace.googleapis.com/v1/projects/{PROJECT}/traces/{trace_id}" + trace_found = False + metric_found = False + while time.monotonic() < deadline and not (trace_found and metric_found): + if not trace_found: + try: + trace_payload = _authorized_get(trace_url, access_token) + trace_found = bool(trace_payload.get("spans")) + except urllib.error.HTTPError as error: + if error.code != 404: + raise + if not metric_found: + try: + metric_payload = _authorized_get(metric_url, access_token) + metric_found = bool(metric_payload.get("timeSeries")) + except urllib.error.HTTPError as error: + if error.code != 404: + raise + if not (trace_found and metric_found): + time.sleep(5) + return { + "trace_stored": trace_found, + "metric_stored": metric_found, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--endpoint", required=True) + parser.add_argument("--service-account", required=True) + parser.add_argument("--timeout-seconds", type=int, default=120) + arguments = parser.parse_args() + + # Add a run identifier without placing high-cardinality values on the metric. + run_id = secrets.token_hex(4) + result = _send_signals(arguments.endpoint, arguments.service_account) + result.update( + _wait_for_storage( + trace_id=str(result["trace_id"]), + metric_type=str(result["metric_type"]), + timeout_seconds=arguments.timeout_seconds, + ) + ) + result["run_id"] = run_id + print(json.dumps(result, sort_keys=True)) + + return ( + 0 + if ( + result["trace_stored"] + and result["metric_stored"] + and result["log_export_status"] != "accepted" + ) + else 1 + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/gcp/workload-inventory.yaml b/deploy/gcp/workload-inventory.yaml new file mode 100644 index 0000000..ce89ae2 --- /dev/null +++ b/deploy/gcp/workload-inventory.yaml @@ -0,0 +1,70 @@ +schema_version: policyengine.observability.workloads.v1 +destination_project: + project_id: policyengine-observability + project_number: "790230211054" + log_bucket: policyengine-observability + log_bucket_location: global + log_retention_days: 30 + +cloud_run: + - service_name: policyengine-api + source_project: policyengine-api + region: us-central1 + environment: production + service_account: policyengine-api-cr-runtime@policyengine-api.iam.gserviceaccount.com + signals: [logs, traces, metrics] + include_tagged_revisions: true + - service_name: policyengine-api-staging + source_project: policyengine-api + region: us-central1 + environment: staging + service_account: policyengine-api-cr-staging@policyengine-api.iam.gserviceaccount.com + signals: [logs, traces, metrics] + include_tagged_revisions: true + - service_name: policyengine-simulation-entry + source_project: policyengine-simulation-entry + region: us-central1 + environment: production + service_account: sim-entry-prod-runtime@policyengine-simulation-entry.iam.gserviceaccount.com + signals: [logs, traces, metrics] + include_tagged_revisions: true + - service_name: policyengine-simulation-entry-staging + source_project: policyengine-simulation-entry + region: us-central1 + environment: staging + service_account: sim-entry-beta-runtime@policyengine-simulation-entry.iam.gserviceaccount.com + signals: [logs, traces, metrics] + include_tagged_revisions: true + +modal: + workspace_name: policyengine + workspace_id: ac-AVO94gmhUEaFCuduiKdpRJ + environments: + - name: main + deployment_environment: production + - name: staging + deployment_environment: staging + applications: + - match: exact + value: policyengine-simulation-gateway + signals: [logs, traces, metrics] + - match: prefix + value: policyengine-simulation-py + name_constraint: '^policyengine-simulation-py[0-9]+-[0-9]+-[0-9]+$' + signals: [logs, traces, metrics] + +excluded: + modal_exact: + - policyengine-simulation-precompute + - policyengine-simulation-executor-smoke + - policyengine-simulation-gateway-smoke + modal_prefixes: + - policyengine-simulation-gateway-ephemeral- + - policyengine-simulation-py4-12-0-ephemeral- + - policyengine-household-api- + - policyengine-uk-chat + - peukchat- + policy: >- + Every Cloud Run service and Modal application absent from the allowlist is + excluded. Excluded application repositories, deployments, and test suites + are outside this change. diff --git a/docs/operations/api-v1-observability.md b/docs/operations/api-v1-observability.md new file mode 100644 index 0000000..cfebaf5 --- /dev/null +++ b/docs/operations/api-v1-observability.md @@ -0,0 +1,153 @@ +# API v1 observability operating policy + +## Scope + +The machine-readable workload inventory is +[`deploy/gcp/workload-inventory.yaml`](../../deploy/gcp/workload-inventory.yaml). +Only the listed `policyengine-api`, simulation entry, simulation gateway, and +versioned simulation executor workloads participate. + +Cloud Run candidate, canary, and tagged revisions use the identity of their +containing service and are included. Modal smoke, precompute, and ephemeral +applications are excluded. `policyengine-household-api` and +`policyengine-uk-chat` remain unchanged and receive no migration or +service-specific validation in this work. + +The inventory is the configuration source for log sink filters, collector +invocation permissions, and the Modal Workload Identity Federation condition. +Telemetry attributes such as `service.namespace` do not grant access. + +## Storage decision + +Production and nonproduction application logs use the existing +`policyengine-observability` bucket in the `policyengine-observability` Google +Cloud project. The bucket is global, analytics enabled, and retains records for +30 days. `deployment.environment.name` distinguishes production and staging. + +A single bucket is appropriate for the initial rollout because the same small +operator group requires both environments, the current bucket already exists, +and a shared analytics surface simplifies request investigations. Access is +controlled at the bucket and project level rather than by environment. This +decision must be revisited before an environment requires different readers, +retention, residency, or deletion policy. + +Cloud Run writes structured JSON to standard output. Exact-service sinks in +the source projects route selected records into this bucket. Modal writes the +same records to standard output and uses the package's bounded asynchronous +Cloud Logging destination under the `policyengine-api-v1-modal` log ID. A +central exclusion prevents a directly ingested record from also being retained +in `_Default`. + +Traces and metrics use Cloud Trace and Cloud Monitoring in the same project. +They are correlated with logs by resource identity, trace ID, request ID, and +job ID; they are not stored in the log bucket. + +## Initial trace sampling + +The initial rollout uses these head-sampling settings: + +| Environment | Services | Parent-based ratio | +| --- | --- | ---: | +| Production | API v1, simulation entry, gateway, executors | 1.0 | +| Staging | API v1, simulation entry, gateway, executors | 1.0 | +| Local development | All | No remote exporter | + +Sampling every initial production trace provides a complete baseline for +volume, cost, errors, and slow operations. After at least one representative +week, operators may lower the normal-request ratio only after recording a cost +and coverage review. A lower head-sampling ratio cannot retroactively retain a +request after its outcome or duration becomes known. Retaining all errors or +slow requests with a lower normal ratio therefore requires a reviewed, +bounded collector tail-sampling policy. + +The checked-in collector configuration implements error and 30-second latency +policies and initially retains 100% of all remaining traces. Any later +reduction applies to the collector's general tail policy while SDK head +sampling remains at 100%, allowing the collector to evaluate completed traces. + +Parent sampling decisions are preserved. Society-wide simulation dispatches +must not override a sampled parent with an unsampled child. + +## Asynchronous trace relationships + +Captured dispatch context contains its UTC capture time. A worker uses the +dispatch span as its parent only when all of these conditions hold: + +- The work is a direct continuation of one dispatch. +- The worker starts no more than five minutes after capture. +- The invocation is not an independent retry. +- The invocation does not aggregate multiple dispatches. + +Otherwise the worker starts a new trace and links the dispatch span. Request +and job identifiers remain the same in either representation. Malformed or +expired remote context is ignored without rejecting the job. + +## Data policy + +### Required log fields + +- `schema_version` +- `timestamp` +- `severity` +- `message` or `event.name` +- `service.name`, `service.namespace`, `service.version`, `service.role` +- `deployment.environment.name` and `cloud.platform` +- Request, operation, trace, span, duration, outcome, and bounded error fields + when applicable + +Application attributes are stored below `attributes`. The initial allowlist is +limited to bounded operational values such as country, model version, backend, +requested version, resolved channel, authentication outcome, job type, and +simulation year. Attribute strings are truncated at 1,024 characters and one +record contains at most 32 application attributes. + +### Trace attributes + +Traces may contain the standard service resource fields, HTTP route templates, +HTTP methods, status codes, operation names, bounded deployment identifiers, +request IDs, job IDs, and explicitly approved operational attributes. Raw URLs, +query values, request bodies, response bodies, and arbitrary baggage are not +recorded. + +### Metric labels + +Metrics use only these bounded labels: + +- `service.name` +- `service.role` +- `deployment.environment.name` +- `cloud.platform` +- `http.route` +- `http.request.method` +- `http.response.status_code_class` +- `operation.name` +- `operation.kind` +- `outcome` + +Request IDs, trace IDs, job IDs, simulation IDs, raw paths, error messages, +user-provided values, unrestricted geography values, and unrestricted version +values are prohibited metric labels. + +### Prohibited telemetry data + +Logs, traces, metrics, and dispatch context must not contain: + +- Authorization headers, cookies, credentials, tokens, or secret values +- Request or response bodies +- Household situations, entity records, or person-level values +- Reform definitions or parameter payloads +- Raw IP addresses +- Prompts, model inputs, or model responses +- Exception local variables +- Function arguments or return values captured automatically + +Exception messages and stacks are truncated and passed through configured +secret-value redaction before remote delivery. + +## Operational limits + +Remote application delivery is best effort. Every application queue, exporter, +retry, network request, flush, and shutdown action has a finite bound. Queue +overflow drops the new record and increments a local counter. Internal +diagnostics are rate limited and written directly to standard error so they do +not recurse through a failing exporter. diff --git a/docs/operations/google-cloud-stage3-runbook.md b/docs/operations/google-cloud-stage3-runbook.md deleted file mode 100644 index c6c25b2..0000000 --- a/docs/operations/google-cloud-stage3-runbook.md +++ /dev/null @@ -1,195 +0,0 @@ -# Google Cloud Stage 3 Observability Runbook - -This runbook documents the fixed Google Cloud Logging destination for -PolicyEngine observability records. - -The public runbook intentionally uses placeholders for live project IDs, -project numbers, service accounts, workload identity provider paths, and sink -writer identities. Keep concrete values in private infrastructure state, -repository/environment secrets, or internal operations documentation. - -## Central Destination - -- Project: `` -- Project number: `` -- Log bucket: `` -- Bucket location: `` -- Bucket retention: 30 days -- Log Analytics: enabled -- Primary log name: `` - -Required APIs: - -```bash -gcloud services enable \ - logging.googleapis.com \ - iamcredentials.googleapis.com \ - sts.googleapis.com \ - --project= -``` - -## Cloud Run Logging - -Cloud Run has two log paths. - -App observability records are written directly by the observability package to -the central project. Cloud Run runtime service accounts need: - -```text -roles/logging.logWriter -``` - -on ``. - -Service accounts should be granted explicitly per deployed service and -environment: - -```text - - -``` - -Native Cloud Run logs are routed with a Cloud Logging sink from -the source application project: - -```text -sink: -destination: logging.googleapis.com/projects//locations//buckets/ -filter: resource.type="cloud_run_revision" -writer: -``` - -The sink writer has `roles/logging.bucketWriter` on -``. - -## Modal Logging - -Modal platform logs stay in Modal for now. Modal simulation container -observability records are written directly to Google Cloud Logging by the -observability package. - -The Modal writer service account is an environment-specific service account: - -```text - -``` - -It has `roles/logging.logWriter` on ``. - -Modal Workload Identity Federation: - -```text -provider: projects//locations/global/workloadIdentityPools//providers/ -issuer: https://oidc.modal.com -audience: oidc.modal.com -``` - -The writer service account grants `roles/iam.workloadIdentityUser` to: - -```text -principalSet://iam.googleapis.com/projects//locations/global/workloadIdentityPools// -``` - -The impersonation grant must be constrained by Modal OIDC claims or by an -equivalent provider condition. Do not use an unconstrained wildcard grant. The -condition should allow only the expected Modal app names and environments for -the services being deployed. - -## Runtime Configuration - -Use these variables for deployed Cloud Run and Modal environments: - -```bash -# Cloud Run: the agent ingests stdout, no direct writes needed. -OBSERVABILITY_LOG_PROFILE=gcp-agent -OBSERVABILITY_GOOGLE_CLOUD_PROJECT= - -# Modal: stdout plus queued direct Cloud Logging writes. -OBSERVABILITY_LOG_PROFILE=gcp-direct -OBSERVABILITY_GOOGLE_CLOUD_PROJECT= -OBSERVABILITY_GOOGLE_CLOUD_LOG_NAME= -OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER=projects//locations/global/workloadIdentityPools//providers/ -OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL= -``` - -Cloud Run does not need the Modal WIF variables. Modal does. - -Do not use `OBSERVABILITY_GOOGLE_LOG_NAME`; the package reads -`OBSERVABILITY_GOOGLE_CLOUD_LOG_NAME`. - -## Smoke Tests - -Write a manual structured log: - -```bash -gcloud logging write \ - '{"event":"stage3_smoke_test","service_name":"","schema_version":"policyengine.observability.smoke.v1"}' \ - --payload-type=json \ - --project= \ - --severity=INFO -``` - -Read it back: - -```bash -gcloud logging read \ - 'logName="projects//logs/" AND jsonPayload.event="stage3_smoke_test"' \ - --project= \ - --limit=1 \ - --format=json -``` - -Find app observability logs: - -```text -logName="projects//logs/" -jsonPayload.service_name="" -``` - -Find Cloud Run app observability logs: - -```text -jsonPayload.service_name="" -jsonPayload.platform="google_cloud_run" -``` - -Find Modal simulation container observability logs: - -```text -jsonPayload.service_name="" -jsonPayload.platform="modal" -jsonPayload.service_role="modal_worker" -``` - -Find native Cloud Run logs routed through the sink: - -```text -resource.type="cloud_run_revision" -``` - -Find observability failures: - -```text -jsonPayload.event="observability_internal_error" -``` - -## Rollback - -For a failing runtime, set the kill switch — plain stdout only, with all -background log machinery disabled: - -```bash -OBSERVABILITY_LOG_PROFILE=plain-sync -``` - -To restore the temporary bridge destination, set: - -```bash -OBSERVABILITY_GOOGLE_CLOUD_PROJECT= -OBSERVABILITY_GOOGLE_CLOUD_LOG_NAME= -OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER= -OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL= -``` - -Leave the central project and sink in place during rollback unless the sink -itself is the source of the problem. diff --git a/policyengine_observability/__init__.py b/policyengine_observability/__init__.py index 538d1f0..c82570e 100644 --- a/policyengine_observability/__init__.py +++ b/policyengine_observability/__init__.py @@ -1,192 +1,44 @@ -from __future__ import annotations - -from typing import Any - -from .config import ObservabilityConfig -from .context import OperationObservabilityContext, RequestObservabilityContext -from .destinations import register_destination, register_stdout_formatter -from .destinations.google_credentials import ( - configure_google_application_credentials, - load_google_credentials, +"""PolicyEngine observability version 2 public interface.""" + +from .adapters import instrument_fastapi, instrument_flask +from .config import ( + DeploymentIdentity, + GoogleCloudLoggingConfig, + LoggingConfig, + ObservabilityConfig, + OTelConfig, + ServiceIdentity, + TelemetryLimits, ) +from .integrations import instrument_httpx from .runtime import ( - OBSERVABILITY_INTERNAL_DISPATCH_HEADER, REQUEST_ID_HEADER, TRACEPARENT_HEADER, + TRACESTATE_HEADER, + ObservabilityLogHandler, ObservabilityRuntime, - observability_runtime, - set_observability_runtime, + configure, + instrument_logging, ) -from .segments import UNKNOWN_SEGMENT, coerce_segment_name - - -def current_context() -> RequestObservabilityContext | None: - return observability_runtime().current_context() - - -def current_operation() -> OperationObservabilityContext | None: - return observability_runtime().current_operation() - - -def set_attribute(key: str, value: Any) -> None: - observability_runtime().set_attribute(key, value) - - -def record_error( - exc: BaseException, - *, - handled: bool, - status_code: int | None = None, - include_stack: bool = True, -) -> None: - observability_runtime().record_error( - exc, - handled=handled, - status_code=status_code, - include_stack=include_stack, - ) - - -def record_event(event: str, **fields: Any) -> None: - observability_runtime().record_event(event, **fields) - - -def traceparent_header() -> str | None: - return observability_runtime().traceparent_header() - - -def capture_context(): - return observability_runtime().capture_context() - - -def mark(key: str, ms: float) -> None: - observability_runtime().mark(key, ms) - - -def mark_ttft(key: str = "ttft_ms") -> None: - observability_runtime().mark_ttft(key) - - -def mark_ttft_attribute(key: str = "ttft_ms") -> None: - observability_runtime().mark_ttft_attribute(key) - - -def start_scope( - timings: dict[str, float], - *, - name: str = "operation", - parent_context: Any = None, - **attrs: Any, -): - return observability_runtime().start_scope( - timings, - name=name, - parent_context=parent_context, - **attrs, - ) - - -def annotate(handle=None, **attrs: Any) -> None: - observability_runtime().annotate(handle, **attrs) - - -def end_scope(handle, error: BaseException | None = None) -> None: - observability_runtime().end_scope(handle, error) - - -def instrument_fastapi(app: Any) -> None: - observability_runtime().instrument_fastapi(app) - - -def instrument_httpx() -> None: - observability_runtime().instrument_httpx() - - -def shutdown_observability() -> None: - observability_runtime().shutdown() - - -def shutdown_tracing() -> None: - shutdown_observability() - - -def restart_observability() -> None: - """Close and rebuild log destinations from configuration. - - For runtimes whose processes fork or restore from memory snapshots - (threads and network clients do not survive either). Call ONLY from - single-threaded lifecycle moments — a post-snapshot-restore hook, a - post-fork hook, before serving traffic. - """ - observability_runtime().restart_log_destinations() - - -def operation(name: str, *, flavor: str | None = None, **attrs: Any): - return observability_runtime().operation(name, flavor=flavor, **attrs) - - -def entrypoint( - name: str | None = None, - *, - flavor: str | None = None, - **attrs: Any, -): - return observability_runtime().entrypoint( - name, - flavor=flavor, - **attrs, - ) - - -def segment(name: Any, **attrs: Any): - return observability_runtime().segment(name, **attrs) - - -def asegment(name: Any, **attrs: Any): - return observability_runtime().asegment(name, **attrs) - - -def collect_timings(name: str = "operation", **attrs: Any): - return observability_runtime().collect_timings(name, **attrs) - +from .schema import SCHEMA_VERSION __all__ = [ - "OBSERVABILITY_INTERNAL_DISPATCH_HEADER", "REQUEST_ID_HEADER", + "SCHEMA_VERSION", "TRACEPARENT_HEADER", - "UNKNOWN_SEGMENT", - "OperationObservabilityContext", + "TRACESTATE_HEADER", + "DeploymentIdentity", + "GoogleCloudLoggingConfig", + "LoggingConfig", + "OTelConfig", "ObservabilityConfig", + "ObservabilityLogHandler", "ObservabilityRuntime", - "RequestObservabilityContext", - "annotate", - "asegment", - "capture_context", - "coerce_segment_name", - "collect_timings", - "configure_google_application_credentials", - "load_google_credentials", - "current_context", - "current_operation", - "end_scope", - "entrypoint", + "ServiceIdentity", + "TelemetryLimits", + "configure", "instrument_fastapi", + "instrument_flask", "instrument_httpx", - "mark", - "mark_ttft", - "mark_ttft_attribute", - "observability_runtime", - "operation", - "record_error", - "record_event", - "register_destination", - "register_stdout_formatter", - "restart_observability", - "segment", - "set_attribute", - "set_observability_runtime", - "shutdown_observability", - "shutdown_tracing", - "start_scope", - "traceparent_header", + "instrument_logging", ] diff --git a/policyengine_observability/_metrics.py b/policyengine_observability/_metrics.py deleted file mode 100644 index ac2d420..0000000 --- a/policyengine_observability/_metrics.py +++ /dev/null @@ -1,188 +0,0 @@ -"""Metric instrument creation and recording.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from .runtime import ObservabilityRuntime - - -class _NoOpInstrument: - def add(self, *_args, **_kwargs) -> None: - return None - - def record(self, *_args, **_kwargs) -> None: - return None - - -class MetricRecorder: - def __init__(self, runtime: ObservabilityRuntime) -> None: - self.runtime = runtime - - def record_operation_metric( - self, - duration_seconds: float, - attributes: dict[str, str], - ) -> None: - try: - self.runtime.operation_duration.record( - duration_seconds, attributes - ) - self.runtime.operations.add(1, attributes) - except BaseException as exc: - self.runtime.log_observability_failure( - "metrics.record_operation", exc - ) - - def record_request_metric( - self, - duration_seconds: float, - attributes: dict[str, str], - ) -> None: - try: - self.runtime.http_duration.record(duration_seconds, attributes) - self.runtime.requests.add(1, attributes) - except BaseException as exc: - self.runtime.log_observability_failure( - "metrics.record_request", exc - ) - - def record_segment_metric( - self, - segment: str, - duration_seconds: float, - attributes: dict[str, str], - *, - backend_segment: bool = False, - ) -> None: - try: - segment_attributes = {**attributes, "segment": segment} - self.runtime.segment_duration.record( - duration_seconds, segment_attributes - ) - if segment == "calculation": - self.runtime.calculate_duration.record( - duration_seconds, attributes - ) - if backend_segment: - self.runtime.backend_duration.record( - duration_seconds, - segment_attributes, - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "metrics.record_segment", - exc, - segment=segment, - ) - - def record_error_metric(self, attributes: dict[str, str]) -> None: - try: - self.runtime.errors.add(1, attributes) - except BaseException as exc: - self.runtime.log_observability_failure("metrics.record_error", exc) - - def record_rate_limited_metric(self, attributes: dict[str, str]) -> None: - try: - self.runtime.rate_limited.add(1, attributes) - except BaseException as exc: - self.runtime.log_observability_failure( - "metrics.record_rate_limited", exc - ) - - def record_failover_event_metric(self, attributes: dict[str, str]) -> None: - try: - self.runtime.failover_events.add(1, attributes) - except BaseException as exc: - self.runtime.log_observability_failure( - "metrics.record_failover_event", - exc, - ) - - def record_active_request( - self, - delta: int, - attributes: dict[str, str], - ) -> None: - try: - self.runtime.active_requests.add(delta, attributes) - except BaseException as exc: - self.runtime.log_observability_failure( - "metrics.add_active_request", exc - ) - - def _configure_instruments(self) -> None: - self.runtime.operation_duration = self.runtime._instrument( - getattr(self.runtime.meter, "create_histogram", None), - "policyengine.operation.duration", - unit="s", - description="PolicyEngine operation duration.", - ) - self.runtime.http_duration = self.runtime._instrument( - getattr(self.runtime.meter, "create_histogram", None), - "http.server.request.duration", - unit="s", - description="HTTP server request duration.", - ) - self.runtime.segment_duration = self.runtime._instrument( - getattr(self.runtime.meter, "create_histogram", None), - "policyengine.segment.duration", - unit="s", - description="PolicyEngine operation segment duration.", - ) - self.runtime.calculate_duration = self.runtime._instrument( - getattr(self.runtime.meter, "create_histogram", None), - "policyengine.calculate.duration", - unit="s", - description="PolicyEngine calculate operation duration.", - ) - self.runtime.backend_duration = self.runtime._instrument( - getattr(self.runtime.meter, "create_histogram", None), - "policyengine.backend.duration", - unit="s", - description="PolicyEngine backend call duration.", - ) - self.runtime.operations = self.runtime._instrument( - getattr(self.runtime.meter, "create_counter", None), - "policyengine.operations", - description="PolicyEngine operation count.", - ) - self.runtime.requests = self.runtime._instrument( - getattr(self.runtime.meter, "create_counter", None), - "policyengine.requests", - description="PolicyEngine request count.", - ) - self.runtime.errors = self.runtime._instrument( - getattr(self.runtime.meter, "create_counter", None), - "policyengine.errors", - description="PolicyEngine error count.", - ) - self.runtime.rate_limited = self.runtime._instrument( - getattr(self.runtime.meter, "create_counter", None), - "policyengine.rate_limited_requests", - description="PolicyEngine rate-limited request count.", - ) - self.runtime.failover_events = self.runtime._instrument( - getattr(self.runtime.meter, "create_counter", None), - "policyengine.failover.events", - description="PolicyEngine failover event count.", - ) - self.runtime.active_requests = self.runtime._instrument( - getattr(self.runtime.meter, "create_up_down_counter", None), - "http.server.active_requests", - description="Active HTTP server requests.", - ) - - def _instrument(self, factory, *args, **kwargs): - if factory is None: - return _NoOpInstrument() - try: - return factory(*args, **kwargs) - except BaseException as exc: - self.runtime.log_observability_failure( - "metrics.create_instrument", - exc, - instrument=args[0] if args else None, - ) - return _NoOpInstrument() diff --git a/policyengine_observability/_operations.py b/policyengine_observability/_operations.py deleted file mode 100644 index 9cefa7b..0000000 --- a/policyengine_observability/_operations.py +++ /dev/null @@ -1,428 +0,0 @@ -"""Operation lifecycle and timing-scope behavior.""" - -from __future__ import annotations - -import inspect -import time -from contextlib import contextmanager -from functools import wraps -from typing import TYPE_CHECKING, Any - -from . import _state -from .context import ( - ErrorRecord, - OperationObservabilityContext, -) - -if TYPE_CHECKING: - from .runtime import ObservabilityRuntime - - -class OperationLifecycle: - def __init__(self, runtime: ObservabilityRuntime) -> None: - self.runtime = runtime - - def operation( - self, - name: str, - *, - flavor: str | None = None, - **attrs: Any, - ): - return _OperationManager( - self.runtime, name, flavor=flavor, attrs=attrs - ) - - def entrypoint( - self, - name: str | None = None, - *, - flavor: str | None = None, - **attrs: Any, - ): - def decorator(func): - operation_name = name or getattr(func, "__name__", "operation") - return self.runtime.operation( - operation_name, - flavor=flavor, - **attrs, - )(func) - - return decorator - - def start_operation( - self, - name: str, - *, - flavor: str | None = None, - parent_context: Any = None, - timings: dict[str, float] | None = None, - emit_log: bool = True, - record_metric: bool = True, - **attrs: Any, - ) -> dict[str, Any]: - handle = { - "operation": None, - "operation_token": None, - "timings_token": None, - "start_token": None, - "context_token": None, - } - if not self.runtime.enabled: - return handle - try: - operation = OperationObservabilityContext( - config=self.runtime.config, - name=self.runtime._safe_str(name), - flavor=flavor, - attributes={ - key: value - for key, value in attrs.items() - if value is not None - }, - timings_ms={}, - emit_log=emit_log, - record_metric=record_metric, - ) - operation.context_token = _state._OPERATION_CONTEXT.set(operation) - handle["operation"] = operation - handle["operation_token"] = operation.context_token - if timings is not None: - handle["timings_token"] = _state._TIMINGS.set(timings) - handle["start_token"] = _state._TURN_START.set(time.perf_counter()) - if parent_context is not None and self.runtime.tracer is not None: - try: - from opentelemetry import context as otel_context - - handle["context_token"] = otel_context.attach( - parent_context - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "operation.context_attach", - exc, - ) - if self.runtime.tracer is not None: - operation.span_handle = self.runtime._start_span( - self.runtime._span_name(operation.name), - operation.span_attributes(), - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "operation.start", exc, name=name - ) - return handle - - def end_operation( - self, - handle: dict[str, Any] | None, - error: BaseException | None = None, - ) -> None: - if not handle: - return - operation = handle.get("operation") - try: - if operation is not None and error is not None: - operation.error = ErrorRecord( - type=type(error).__name__, - message=self.runtime._safe_str(error), - handled=False, - stack=self.runtime._safe_traceback(error), - ) - self.runtime.record_error_metric( - operation.metric_attributes( - error_type=type(error).__name__ - ) - ) - if operation is not None: - self.runtime.complete_operation(operation) - if operation is not None: - self.runtime._end_span(operation.span_handle, error) - except BaseException as exc: - self.runtime.log_observability_failure("operation.end", exc) - finally: - context_token = handle.get("context_token") - if context_token is not None: - try: - from opentelemetry import context as otel_context - - otel_context.detach(context_token) - except BaseException as exc: - self.runtime.log_observability_failure( - "operation.context_detach", - exc, - ) - for var, key in ( - (_state._TIMINGS, "timings_token"), - (_state._TURN_START, "start_token"), - (_state._OPERATION_CONTEXT, "operation_token"), - ): - token = handle.get(key) - if token is not None: - try: - var.reset(token) - except BaseException as exc: - self.runtime.log_observability_failure( - "operation.context_reset", - exc, - token=key, - ) - - def complete_operation( - self, - operation: OperationObservabilityContext, - ) -> None: - if operation.metric_recorded: - return - operation.metric_recorded = True - if operation.record_metric: - self.runtime.record_operation_metric( - operation.duration_seconds(), - operation.metric_attributes(), - ) - if operation.emit_log: - self.runtime.emit_operation_log(operation) - - @contextmanager - def collect_timings(self, name: str = "operation", **attrs: Any): - timings: dict[str, float] = {} - handle = self.runtime.start_scope(timings, name=name, **attrs) - error: BaseException | None = None - try: - yield timings - except BaseException as exc: - error = exc - raise - finally: - self.runtime.end_scope(handle, error) - - def start_scope( - self, - timings: dict[str, float], - *, - name: str = "operation", - parent_context: Any = None, - **attrs: Any, - ) -> dict[str, Any]: - if self.runtime.current_operation() is None: - return { - "operation_handle": self.runtime.start_operation( - name, - parent_context=parent_context, - timings=timings, - **attrs, - ) - } - handle = { - "operation_handle": None, - "timings_token": None, - "start_token": None, - "context_token": None, - "span": None, - } - try: - handle["timings_token"] = _state._TIMINGS.set(timings) - except BaseException as exc: - self.runtime.log_observability_failure("scope.timings_set", exc) - try: - handle["start_token"] = _state._TURN_START.set(time.perf_counter()) - except BaseException as exc: - self.runtime.log_observability_failure("scope.start_set", exc) - if parent_context is not None and self.runtime.tracer is not None: - try: - from opentelemetry import context as otel_context - - handle["context_token"] = otel_context.attach(parent_context) - except BaseException as exc: - self.runtime.log_observability_failure( - "scope.context_attach", exc - ) - try: - if self.runtime.tracer is not None: - handle["span"] = self.runtime._start_span(name, attrs) - except BaseException as exc: - self.runtime.log_observability_failure( - "scope.span_start", exc, span=name - ) - handle["span"] = None - return handle - - def annotate( - self, - handle: dict[str, Any] | None = None, - **attrs: Any, - ) -> None: - try: - if handle: - span_handle = handle.get("span") - if span_handle is not None: - _cm, span = span_handle - for key, value in attrs.items(): - if value is not None: - span.set_attribute(key, value) - context = self.runtime.current_context() - if context is not None: - for key, value in attrs.items(): - context.set_attribute(key, value) - operation = self.runtime.current_operation() - if operation is not None: - for key, value in attrs.items(): - operation.set_attribute(key, value) - self.runtime._set_current_span_attributes( - operation.span_attributes() - ) - except BaseException as exc: - self.runtime.log_observability_failure("scope.annotate", exc) - - def end_scope( - self, - handle: dict[str, Any] | None, - error: BaseException | None = None, - ) -> None: - if not handle: - return - operation_handle = handle.get("operation_handle") - if operation_handle is not None: - self.runtime.end_operation(operation_handle, error) - return - try: - self.runtime._end_span(handle.get("span"), error) - except BaseException as exc: - self.runtime.log_observability_failure("scope.span_end", exc) - context_token = handle.get("context_token") - if context_token is not None: - try: - from opentelemetry import context as otel_context - - otel_context.detach(context_token) - except BaseException as exc: - self.runtime.log_observability_failure( - "scope.context_detach", exc - ) - for var, key in ( - (_state._TIMINGS, "timings_token"), - (_state._TURN_START, "start_token"), - ): - token = handle.get(key) - if token is not None: - try: - var.reset(token) - except BaseException as exc: - self.runtime.log_observability_failure( - "scope.context_reset", - exc, - token=key, - ) - - def mark(self, key: str, ms: float) -> None: - try: - timings = _state._TIMINGS.get() - if timings is not None: - timings[key] = round(float(ms), 1) - except BaseException as exc: - self.runtime.log_observability_failure("scope.mark", exc, key=key) - - def mark_ttft(self, key: str = "ttft_ms") -> None: - try: - start = _state._TURN_START.get() - if start is not None: - self.runtime.mark(key, (time.perf_counter() - start) * 1000.0) - except BaseException as exc: - self.runtime.log_observability_failure("scope.mark_ttft", exc) - - def mark_ttft_attribute(self, key: str = "ttft_ms") -> None: - try: - start = _state._TURN_START.get() - if start is None: - return - self.runtime.annotate( - **{key: round((time.perf_counter() - start) * 1000.0, 1)} - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "scope.mark_ttft_attribute", exc - ) - - def _start_implicit_operation( - self, - segment_name: str, - attrs: dict[str, Any], - ) -> dict[str, Any] | None: - if ( - self.runtime.current_operation() is not None - or self.runtime.current_context() is not None - ): - return None - operation_name = attrs.get("operation") or segment_name - flavor = attrs.get("flavor") - operation_attrs = { - key: value - for key, value in attrs.items() - if key not in {"operation", "flavor"} and value is not None - } - return self.runtime.start_operation( - self.runtime._safe_str(operation_name), - flavor=self.runtime._safe_str(flavor) - if flavor is not None - else None, - **operation_attrs, - ) - - -class _OperationManager: - def __init__( - self, - runtime: ObservabilityRuntime, - name: str, - *, - flavor: str | None, - attrs: dict[str, Any], - ) -> None: - self.runtime = runtime - self.name = name - self.flavor = flavor - self.attrs = attrs - self.handle: dict[str, Any] | None = None - - def __enter__(self): - self.handle = self.runtime.start_operation( - self.name, - flavor=self.flavor, - **self.attrs, - ) - return self.runtime.current_operation() - - def __exit__(self, exc_type, exc, _traceback) -> bool: - self.runtime.end_operation(self.handle, exc) - return False - - async def __aenter__(self): - return self.__enter__() - - async def __aexit__(self, exc_type, exc, traceback) -> bool: - return self.__exit__(exc_type, exc, traceback) - - def __call__(self, func): - if inspect.iscoroutinefunction(func): - - @wraps(func) - async def async_wrapper(*args, **kwargs): - async with self.runtime.operation( - self.name, - flavor=self.flavor, - **self.attrs, - ): - return await func(*args, **kwargs) - - return async_wrapper - - @wraps(func) - def wrapper(*args, **kwargs): - with self.runtime.operation( - self.name, - flavor=self.flavor, - **self.attrs, - ): - return func(*args, **kwargs) - - return wrapper diff --git a/policyengine_observability/_requests.py b/policyengine_observability/_requests.py deleted file mode 100644 index b326e91..0000000 --- a/policyengine_observability/_requests.py +++ /dev/null @@ -1,291 +0,0 @@ -"""Request lifecycle, response metadata, and cleanup.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -from . import _state -from ._state import REQUEST_ID_HEADER, TRACEPARENT_HEADER -from .context import ( - OperationObservabilityContext, - RequestObservabilityContext, -) - -if TYPE_CHECKING: - from .runtime import ObservabilityRuntime - - -class RequestLifecycle: - def __init__(self, runtime: ObservabilityRuntime) -> None: - self.runtime = runtime - - def begin_request( - self, - context: RequestObservabilityContext, - *, - carrier: Any = None, - ) -> None: - if not self.runtime.enabled: - return - try: - context.context_token = _state._REQUEST_CONTEXT.set(context) - context.set_attribute("endpoint", context.endpoint) - self.runtime._begin_request_operation(context) - self.runtime._start_request_span(context, carrier=carrier) - self.runtime.record_active_request(1, context.metric_attributes()) - except BaseException as exc: - self.runtime.log_observability_failure("request.begin", exc) - - def _begin_request_operation( - self, - context: RequestObservabilityContext, - ) -> None: - try: - parent_operation = _state._OPERATION_CONTEXT.get() - timings = context.timings_ms - timing_counts = context.timing_counts - segment_tree = context.segment_tree - segment_sequence = context.segment_sequence - if context.internal_dispatch and parent_operation is not None: - timings = parent_operation.timings_ms - timing_counts = parent_operation.timing_counts - segment_tree = parent_operation.segment_tree - segment_sequence = parent_operation.segment_sequence - context.timings_ms = timings - context.timing_counts = timing_counts - context.segment_tree = segment_tree - context.segment_sequence = segment_sequence - operation = OperationObservabilityContext( - config=context.config, - name=context.route, - flavor="http", - attributes={ - "route": context.route, - "method": context.method, - "endpoint": context.endpoint, - "path": context.path, - }, - timings_ms=timings, - timing_counts=timing_counts, - segment_tree=segment_tree, - segment_sequence=segment_sequence, - emit_log=False, - record_metric=False, - ) - operation.context_token = _state._OPERATION_CONTEXT.set(operation) - context.operation_context = operation - context.operation_token = operation.context_token - except BaseException as exc: - self.runtime.log_observability_failure( - "request.operation_begin", - exc, - request_id=getattr(context, "request_id", None), - ) - - def finish_request(self, status_code: int) -> dict[str, str]: - headers = self.runtime.prepare_response(status_code) - self.runtime.complete_request(status_code) - return headers - - def prepare_response(self, status_code: int) -> dict[str, str]: - if not self.runtime.enabled: - return {} - headers: dict[str, str] = {} - try: - context = self.runtime.current_context() - if context is None: - return headers - context.status_code = status_code - self.runtime._set_current_span_attributes( - context.span_attributes() - ) - if context.operation_context is not None: - context.operation_context.set_attribute( - "status_code", - str(status_code), - ) - headers[REQUEST_ID_HEADER] = context.request_id - traceparent = self.runtime.traceparent_header() - if traceparent: - headers[TRACEPARENT_HEADER] = traceparent - if status_code == 429: - context.set_attribute("rate_limited", True) - return headers - except BaseException as exc: - self.runtime.log_observability_failure( - "request.prepare_response", exc - ) - return headers - - def complete_request(self, status_code: int | None = None) -> None: - if not self.runtime.enabled: - return - try: - context = self.runtime.current_context() - if context is None: - return - if status_code is not None: - context.status_code = status_code - self.runtime._set_current_span_attributes( - context.span_attributes() - ) - if context.request_metric_recorded: - return - context.request_metric_recorded = True - if context.status_code == 429: - self.runtime.record_rate_limited_metric( - context.metric_attributes() - ) - self.runtime.record_request_metric( - context.duration_seconds(), - context.metric_attributes(), - ) - self.runtime._close_active_request(context) - except BaseException as exc: - self.runtime.log_observability_failure("request.complete", exc) - - def update_request_route( - self, - *, - route: str | None = None, - endpoint: str | None = None, - ) -> None: - if not self.runtime.enabled: - return - try: - context = self.runtime.current_context() - if context is None: - return - route_changed = bool(route and route != context.route) - old_active_attributes = ( - context.metric_attributes() - if route_changed and not context.active_closed - else None - ) - if route: - context.route = route - if context.operation_context is not None: - context.operation_context.name = route - context.operation_context.set_attribute("route", route) - if endpoint: - context.endpoint = endpoint - context.set_attribute("endpoint", endpoint) - if context.operation_context is not None: - context.operation_context.set_attribute( - "endpoint", - endpoint, - ) - self.runtime._set_current_span_attributes( - context.span_attributes() - ) - span = context.server_span - update_name = getattr(span, "update_name", None) - if route and update_name is not None: - update_name(route) - if old_active_attributes is not None: - self.runtime.record_active_request(-1, old_active_attributes) - self.runtime.record_active_request( - 1, context.metric_attributes() - ) - except BaseException as exc: - self.runtime.log_observability_failure("request.update_route", exc) - - def teardown_request(self, exc: BaseException | None = None) -> None: - if not self.runtime.enabled: - return - context = self.runtime.current_context() - if context is None: - return - try: - if exc is not None: - self.runtime.record_error( - exc, - handled=False, - status_code=context.status_code or 500, - ) - self.runtime._close_active_request(context) - self.runtime.emit_request_log(context) - except BaseException as observability_exc: - self.runtime.log_observability_failure( - "request.teardown", - observability_exc, - ) - finally: - self.runtime._close_request_span(context, exc) - self.runtime._reset_request_operation_context(context) - self.runtime._reset_request_context(context) - - def set_attribute(self, key: str, value: Any) -> None: - if not self.runtime.enabled: - return - try: - context = self.runtime.current_context() - if context is not None: - context.set_attribute(key, value) - if context.operation_context is not None: - context.operation_context.set_attribute(key, value) - self.runtime._set_current_span_attributes( - context.span_attributes(**{f"policyengine.{key}": value}) - ) - operation = self.runtime.current_operation() - if operation is not None and operation is not getattr( - context, "operation_context", None - ): - operation.set_attribute(key, value) - self.runtime._set_current_span_attributes( - operation.span_attributes(**{f"policyengine.{key}": value}) - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "request.set_attribute", - exc, - attribute=key, - ) - - def _close_active_request( - self, - context: RequestObservabilityContext, - ) -> None: - try: - if context.active_closed: - return - context.active_closed = True - self.runtime.record_active_request(-1, context.metric_attributes()) - except BaseException as exc: - self.runtime.log_observability_failure( - "request.close_active", - exc, - request_id=getattr(context, "request_id", None), - ) - - def _reset_request_operation_context( - self, - context: RequestObservabilityContext, - ) -> None: - token = context.operation_token - if token is None: - return - try: - _state._OPERATION_CONTEXT.reset(token) - except BaseException as exc: - self.runtime.log_observability_failure( - "request.operation_context_reset", - exc, - request_id=getattr(context, "request_id", None), - ) - - def _reset_request_context( - self, - context: RequestObservabilityContext, - ) -> None: - token = context.context_token - if token is None: - return - try: - _state._REQUEST_CONTEXT.reset(token) - except BaseException as exc: - self.runtime.log_observability_failure( - "request.context_reset", - exc, - request_id=getattr(context, "request_id", None), - ) diff --git a/policyengine_observability/_state.py b/policyengine_observability/_state.py deleted file mode 100644 index a601d3b..0000000 --- a/policyengine_observability/_state.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Shared context variables for requests, operations, and segments.""" - -from __future__ import annotations - -from contextvars import ContextVar -from typing import TYPE_CHECKING - -from .context import ( - OperationObservabilityContext, - RequestObservabilityContext, - SegmentTimingNode, -) - -if TYPE_CHECKING: - from .runtime import ObservabilityRuntime - -OBSERVABILITY_INTERNAL_DISPATCH_HEADER = "X-PolicyEngine-Internal-Dispatch" -REQUEST_ID_HEADER = "X-PolicyEngine-Request-Id" -TRACEPARENT_HEADER = "traceparent" - -_REQUEST_CONTEXT: ContextVar[RequestObservabilityContext | None] = ContextVar( - "policyengine_request_observability_context", - default=None, -) -_OPERATION_CONTEXT: ContextVar[OperationObservabilityContext | None] = ( - ContextVar( - "policyengine_operation_observability_context", - default=None, - ) -) -_TIMINGS: ContextVar[dict[str, float] | None] = ContextVar( - "policyengine_observability_timings", - default=None, -) -_TURN_START: ContextVar[float | None] = ContextVar( - "policyengine_observability_turn_start", - default=None, -) -_SEGMENT_STACK: ContextVar[tuple[tuple[int, SegmentTimingNode], ...]] = ( - ContextVar( - "policyengine_observability_segment_stack", - default=(), - ) -) - - -class ContextState: - def __init__(self, runtime: ObservabilityRuntime) -> None: - self.runtime = runtime - - def current_context(self) -> RequestObservabilityContext | None: - try: - return _REQUEST_CONTEXT.get() - except BaseException as exc: - self.runtime.log_observability_failure("context.current", exc) - return None - - def current_operation( - self, - ) -> OperationObservabilityContext | None: - try: - return _OPERATION_CONTEXT.get() - except BaseException as exc: - self.runtime.log_observability_failure("operation.current", exc) - return None diff --git a/policyengine_observability/_tracing.py b/policyengine_observability/_tracing.py deleted file mode 100644 index b8dfc6c..0000000 --- a/policyengine_observability/_tracing.py +++ /dev/null @@ -1,434 +0,0 @@ -"""Trace initialization, propagation, and span lifecycle.""" - -from __future__ import annotations - -from collections.abc import Iterator -from contextlib import contextmanager -from typing import TYPE_CHECKING, Any - -from ._state import TRACEPARENT_HEADER -from .context import ( - RequestObservabilityContext, -) - -if TYPE_CHECKING: - from .runtime import ObservabilityRuntime - - -def _is_safe_span_value(value: Any) -> bool: - return isinstance(value, str | bool | int | float) - - -class TraceRecorder: - def __init__(self, runtime: ObservabilityRuntime) -> None: - self.runtime = runtime - - def traceparent_header(self) -> str | None: - if not self.runtime.enabled or self.runtime.propagate is None: - return None - try: - carrier: dict[str, str] = {} - self.runtime.propagate.inject(carrier) - return carrier.get(TRACEPARENT_HEADER) - except BaseException as exc: - self.runtime.log_observability_failure( - "request.traceparent_header", exc - ) - return None - - def capture_context(self): - if self.runtime.tracer is None: - return None - try: - from opentelemetry import context as otel_context - - return otel_context.get_current() - except BaseException as exc: - self.runtime.log_observability_failure("otel.capture_context", exc) - return None - - def instrument_fastapi(self, app: Any) -> None: - if not self.runtime.enabled or not self.runtime.config.otel_enabled: - return - try: - from opentelemetry.instrumentation.fastapi import ( - FastAPIInstrumentor, - ) - - FastAPIInstrumentor.instrument_app(app) - except BaseException as exc: - self.runtime.log_observability_failure( - "fastapi.auto_instrument", - exc, - ) - - def instrument_httpx(self) -> None: - if ( - not self.runtime.enabled - or not self.runtime.config.otel_enabled - or self.runtime._httpx_instrumented - ): - return - try: - from opentelemetry.instrumentation.httpx import ( - HTTPXClientInstrumentor, - ) - - HTTPXClientInstrumentor().instrument() - self.runtime._httpx_instrumented = True - except BaseException as exc: - self.runtime.log_observability_failure( - "httpx.auto_instrument", exc - ) - - def _configure_otel(self) -> None: - try: - from opentelemetry import metrics, propagate, trace - from opentelemetry.sdk.metrics import MeterProvider - from opentelemetry.sdk.resources import ( - DEPLOYMENT_ENVIRONMENT, - SERVICE_NAME, - Resource, - ) - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.trace import SpanKind, Status, StatusCode - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.configure_imports", exc - ) - return - - try: - resource = Resource.create( - { - SERVICE_NAME: self.runtime.config.service_name, - DEPLOYMENT_ENVIRONMENT: self.runtime.config.environment, - "service.role": self.runtime.config.service_role, - } - ) - tracer_provider = TracerProvider(resource=resource) - metric_readers = [] - if self.runtime.config.otlp_endpoint: - self.runtime._add_trace_exporter(tracer_provider) - metric_reader = self.runtime._metric_reader() - if metric_reader is not None: - metric_readers.append(metric_reader) - self.runtime.tracer_provider = tracer_provider - try: - trace.set_tracer_provider(tracer_provider) - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.set_tracer_provider", - exc, - ) - try: - self.runtime.meter_provider = MeterProvider( - resource=resource, - metric_readers=metric_readers, - ) - metrics.set_meter_provider(self.runtime.meter_provider) - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.set_meter_provider", - exc, - ) - self.runtime.trace = trace - self.runtime.propagate = propagate - self.runtime.SpanKind = SpanKind - self.runtime.Status = Status - self.runtime.StatusCode = StatusCode - tracer_name = ( - self.runtime.config.tracer_name - or self.runtime.config.service_name - ) - meter_name = ( - self.runtime.config.meter_name - or self.runtime.config.service_name - ) - self.runtime.tracer = trace.get_tracer(tracer_name) - self.runtime.meter = metrics.get_meter(meter_name) - self.runtime._configure_instruments() - except BaseException as exc: - self.runtime.log_observability_failure("otel.configure", exc) - - def _add_trace_exporter(self, tracer_provider) -> None: - try: - if self.runtime.config.otlp_protocol.startswith("http"): - from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( - OTLPSpanExporter, - ) - else: - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( - OTLPSpanExporter, - ) - from opentelemetry.sdk.trace.export import BatchSpanProcessor - - tracer_provider.add_span_processor( - BatchSpanProcessor(OTLPSpanExporter()) - ) - except BaseException as exc: - self.runtime.log_observability_failure("otel.trace_exporter", exc) - - def _metric_reader(self): - try: - if self.runtime.config.otlp_protocol.startswith("http"): - from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( - OTLPMetricExporter, - ) - else: - from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( - OTLPMetricExporter, - ) - from opentelemetry.sdk.metrics.export import ( - PeriodicExportingMetricReader, - ) - - return PeriodicExportingMetricReader(OTLPMetricExporter()) - except BaseException as exc: - self.runtime.log_observability_failure("otel.metric_exporter", exc) - return None - - def _start_request_span( - self, - context: RequestObservabilityContext, - *, - carrier: Any = None, - ) -> None: - if self.runtime.tracer is None: - return - attrs = context.span_attributes() - parent_context = self.runtime._extract_context(carrier) - try: - context.server_span_cm = self.runtime.tracer.start_as_current_span( - context.route, - context=parent_context, - kind=self.runtime.SpanKind.SERVER - if self.runtime.SpanKind - else None, - attributes=attrs, - ) - context.server_span = context.server_span_cm.__enter__() - except BaseException as exc: - context.server_span_cm = None - context.server_span = None - self.runtime.log_observability_failure( - "otel.request_span_enter", exc - ) - - def _close_request_span( - self, - context: RequestObservabilityContext, - exc: BaseException | None, - ) -> None: - if context.span_closed: - return - context.span_closed = True - span_cm = context.server_span_cm - if span_cm is None: - return - try: - if exc is None: - span_cm.__exit__(None, None, None) - else: - span_cm.__exit__(type(exc), exc, exc.__traceback__) - except BaseException as observability_exc: - self.runtime.log_observability_failure( - "otel.request_span_exit", - observability_exc, - request_id=context.request_id, - ) - - @contextmanager - def _safe_span(self, name: str, attrs: dict[str, Any]) -> Iterator[Any]: - if self.runtime.tracer is None: - yield None - return - span_handle = self.runtime._start_span(name, attrs) - if span_handle is None: - yield None - return - _cm, span = span_handle - try: - yield span - except BaseException as exc: - try: - self.runtime._end_span(span_handle, exc) - except BaseException as observability_exc: - self.runtime.log_observability_failure( - "otel.span_exit", - observability_exc, - span=name, - ) - raise - else: - try: - self.runtime._end_span(span_handle) - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.span_exit", - exc, - span=name, - ) - - def _start_span(self, name: str, attrs: dict[str, Any]): - try: - span_cm = self.runtime.tracer.start_as_current_span(name) - span = span_cm.__enter__() - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.span_enter", exc, span=name - ) - return None - try: - for key, value in attrs.items(): - if value is not None: - span.set_attribute(key, value) - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.span_attributes", - exc, - span=name, - ) - return span_cm, span - - def _end_span( - self, - span_handle, - error: BaseException | None = None, - ) -> None: - if span_handle is None: - return - span_cm, span = span_handle - try: - if error is not None: - self.runtime._record_exception_on_span( - span, - error, - handled=False, - status_code=500, - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.span_error_status", exc - ) - try: - span_cm.__exit__(None, None, None) - except BaseException as exc: - self.runtime.log_observability_failure("otel.span_exit", exc) - - def _segment_span_attributes( - self, - attrs: dict[str, Any], - ) -> dict[str, Any]: - context = self.runtime.current_context() - operation = self.runtime.current_operation() - span_attrs = { - key: value for key, value in attrs.items() if value is not None - } - if context is not None: - span_attrs = {**context.span_attributes(), **span_attrs} - elif operation is not None: - span_attrs = {**operation.span_attributes(), **span_attrs} - return span_attrs - - def _span_name(self, segment_name: str) -> str: - if not self.runtime.config.span_prefix: - return segment_name - return f"{self.runtime.config.span_prefix}.{segment_name}" - - def _set_current_span_attributes(self, attrs: dict[str, Any]) -> None: - span = self.runtime._current_span() - if span is None: - return - try: - for key, value in attrs.items(): - if value is not None: - span.set_attribute(key, value) - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.set_span_attributes", exc - ) - - def _current_span(self): - if self.runtime.trace is None: - return None - try: - return self.runtime.trace.get_current_span() - except BaseException as exc: - self.runtime.log_observability_failure("otel.current_span", exc) - return None - - def _trace_ids(self) -> tuple[str | None, str | None]: - span = self.runtime._current_span() - if span is None: - return None, None - try: - context = span.get_span_context() - except BaseException as exc: - self.runtime.log_observability_failure("otel.span_context", exc) - return None, None - if not getattr(context, "is_valid", False): - return None, None - return f"{context.trace_id:032x}", f"{context.span_id:016x}" - - def _extract_context(self, carrier: Any): - if self.runtime.propagate is None or carrier is None: - return None - try: - return self.runtime.propagate.extract(carrier) - except BaseException as exc: - self.runtime.log_observability_failure("otel.extract_context", exc) - return None - - def _record_exception_on_span( - self, - span, - exc: BaseException, - *, - handled: bool, - status_code: int | None, - ) -> None: - try: - span.record_exception(exc) - span.set_attribute("error.type", type(exc).__name__) - span.set_attribute("error.handled", handled) - if ( - self.runtime.Status is not None - and self.runtime.StatusCode is not None - and ( - not handled - or (status_code is not None and status_code >= 500) - ) - ): - span.set_status( - self.runtime.Status( - self.runtime.StatusCode.ERROR, - self.runtime._safe_str(exc), - ) - ) - except BaseException as observability_exc: - self.runtime.log_observability_failure( - "otel.record_exception", - observability_exc, - original_error_type=type(exc).__name__, - ) - - def _add_span_event(self, event: str, fields: dict[str, Any]) -> None: - span = self.runtime._current_span() - if span is None: - return - try: - span.add_event( - event, - { - key: value - for key, value in fields.items() - if _is_safe_span_value(value) - }, - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.add_event", - exc, - event_name=event, - ) diff --git a/policyengine_observability/adapters/__init__.py b/policyengine_observability/adapters/__init__.py index 99cc2ad..96a7008 100644 --- a/policyengine_observability/adapters/__init__.py +++ b/policyengine_observability/adapters/__init__.py @@ -1 +1,6 @@ """Framework adapters for policyengine-observability.""" + +from .fastapi import instrument_fastapi +from .flask import instrument_flask + +__all__ = ["instrument_fastapi", "instrument_flask"] diff --git a/policyengine_observability/adapters/fastapi.py b/policyengine_observability/adapters/fastapi.py index 91fc0f2..c5817ee 100644 --- a/policyengine_observability/adapters/fastapi.py +++ b/policyengine_observability/adapters/fastapi.py @@ -1,302 +1,137 @@ from __future__ import annotations -import uuid +from collections.abc import Awaitable, Callable from typing import Any -from urllib.parse import parse_qs -from ..config import ObservabilityConfig -from ..context import RequestObservabilityContext -from ..runtime import ( - REQUEST_ID_HEADER, - TRACEPARENT_HEADER, - ObservabilityRuntime, - set_observability_runtime, -) +from ..runtime import ObservabilityRuntime -UNMATCHED_ROUTE = "" +_STATE_KEY = "policyengine_observability" -class FastAPIObservabilityAdapter: - def __init__( - self, - runtime: ObservabilityRuntime, - *, - static_attributes: dict[str, Any] | None = None, - ) -> None: - self.runtime = runtime - self.static_attributes = { - key: value - for key, value in (static_attributes or {}).items() - if value is not None - } +def instrument_fastapi( + app: Any, runtime: ObservabilityRuntime +) -> ObservabilityRuntime: + """Install one request lifecycle integration on a FastAPI application.""" - def instrument_app(self, app: Any) -> None: - if not self.runtime.enabled: - return - if getattr(app.state, "policyengine_observability_adapter", None): - return - app.state.policyengine_observability_adapter = self - if self.runtime.config.instrument_fastapi: - self.runtime.instrument_fastapi(app) - try: - app.add_middleware( - FastAPIObservabilityMiddleware, - adapter=self, - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "fastapi.middleware_install", - exc, - ) + existing = getattr(app.state, _STATE_KEY, None) + if isinstance(existing, ObservabilityRuntime): + return existing + try: + app.add_middleware(_ObservabilityMiddleware, runtime=runtime) + setattr(app.state, _STATE_KEY, runtime) + except Exception as exc: + runtime.diagnostics.report("fastapi.middleware_install", exc) + return runtime - def start_request(self, scope: dict[str, Any]) -> None: - try: - headers = _headers_from_scope(scope) - path = scope.get("path") or "" - route = _route_from_scope(scope) or UNMATCHED_ROUTE - endpoint = _endpoint_from_scope(scope) - request_id = headers.get(REQUEST_ID_HEADER.lower()) or str( - uuid.uuid4() - ) - context = RequestObservabilityContext( - config=self.runtime.config, - request_id=request_id, - method=scope.get("method") or "", - route=route, - path=path, - endpoint=endpoint, - query_keys=_query_keys(scope), - content_length_bytes=_int_header( - headers.get("content-length") - ), - inbound=self._inbound_metadata(scope, headers), - ) - self.runtime.begin_request(context, carrier=headers) - if self.static_attributes: - self.runtime.annotate(**self.static_attributes) - except BaseException as exc: - self.runtime.log_observability_failure( - "fastapi.before_request", - exc, - ) - def update_resolved_route(self, scope: dict[str, Any]) -> None: - route = _route_from_scope(scope) - endpoint = _endpoint_from_scope(scope) - if route or endpoint: - self.runtime.update_request_route(route=route, endpoint=endpoint) +class _ObservabilityMiddleware: + def __init__(self, app: Any, *, runtime: ObservabilityRuntime) -> None: + self.app = app + self.runtime = runtime - def _inbound_metadata( + async def __call__( self, scope: dict[str, Any], - headers: dict[str, str], - ) -> dict[str, Any]: - forwarded_for = _split_forwarded_for(headers.get("x-forwarded-for")) - x_real_ip = headers.get("x-real-ip") - client = scope.get("client") or () - remote_addr = client[0] if client else None - client_ip = None - ip_source = None - if forwarded_for: - client_ip = forwarded_for[0] - ip_source = "x_forwarded_for" - elif x_real_ip: - client_ip = x_real_ip - ip_source = "x_real_ip" - elif remote_addr: - client_ip = remote_addr - ip_source = "remote_addr" - metadata = { - "ip_source": ip_source, - "user_agent": headers.get("user-agent"), - "origin": headers.get("origin"), - "referer": headers.get("referer"), - "host": headers.get("host"), - "content_length_bytes": _int_header(headers.get("content-length")), - } - if self.runtime.config.log_raw_ip: - metadata["client_ip"] = client_ip - metadata["forwarded_for"] = forwarded_for - metadata["x_real_ip"] = x_real_ip - return metadata - - -class FastAPIObservabilityMiddleware: - def __init__( - self, - app: Any, - *, - adapter: FastAPIObservabilityAdapter, + receive: Callable[[], Awaitable[dict[str, Any]]], + send: Callable[[dict[str, Any]], Awaitable[None]], ) -> None: - self.app = app - self.adapter = adapter - - async def __call__(self, scope, receive, send) -> None: if scope.get("type") != "http": await self.app(scope, receive, send) return - self.adapter.start_request(scope) status_code: int | None = None + error: Exception | None = None completed = False + try: + self.runtime.begin_request( + headers=_headers_from_scope(scope), + method=str(scope.get("method") or ""), + route=_route_from_scope(scope) or "", + ) + except Exception as exc: + self.runtime.diagnostics.report("fastapi.request_begin", exc) - async def send_wrapper(message) -> None: - nonlocal completed - nonlocal status_code - - if message["type"] == "http.response.start": + async def send_observed(message: dict[str, Any]) -> None: + nonlocal completed, status_code + if message.get("type") == "http.response.start": status_code = int(message.get("status") or 0) - self.adapter.update_resolved_route(scope) - response_headers = self.adapter.runtime.prepare_response( - status_code - ) - if response_headers: - message = { - **message, - "headers": _merge_response_headers( - message.get("headers") or [], - response_headers, - ), - } - await send(message) - return - - if message["type"] == "http.response.body" and not message.get( + self._update_route(scope) + message = { + **message, + "headers": _merge_headers( + list(message.get("headers") or []), + self.runtime.response_headers(), + ), + } + await send(message) + if message.get("type") == "http.response.body" and not message.get( "more_body", False ): - try: - await send(message) - finally: - completed = True - self.adapter.update_resolved_route(scope) - self.adapter.runtime.complete_request(status_code) - self.adapter.runtime.teardown_request(None) - return - - await send(message) + self._update_route(scope) + self._end(status_code=status_code) + completed = True try: - await self.app(scope, receive, send_wrapper) - except BaseException as exc: - if not completed: - error_status = status_code or 500 - self.adapter.update_resolved_route(scope) - self.adapter.runtime.prepare_response(error_status) - self.adapter.runtime.complete_request(error_status) - self.adapter.runtime.teardown_request(exc) - completed = True + await self.app(scope, receive, send_observed) + except Exception as exc: + error = exc raise finally: if not completed: - self.adapter.update_resolved_route(scope) - self.adapter.runtime.complete_request(status_code) - self.adapter.runtime.teardown_request(None) + self._update_route(scope) + self._end( + status_code=status_code or (500 if error else None), + error=error, + ) + def _update_route(self, scope: dict[str, Any]) -> None: + try: + route = _route_from_scope(scope) + if route: + self.runtime.update_request_route(route) + except Exception as exc: + self.runtime.diagnostics.report("fastapi.route_update", exc) -def init_fastapi_observability( - app: Any, - *, - config: ObservabilityConfig | None = None, - runtime: ObservabilityRuntime | None = None, - service_name: str, - service_role: str = "api", - span_prefix: str | None = None, - segment_registry=None, - static_attributes: dict[str, Any] | None = None, -) -> ObservabilityRuntime: - existing = getattr(app.state, "policyengine_observability", None) - if existing: - return existing - runtime = runtime or ObservabilityRuntime( - config - or ObservabilityConfig.from_env( - service_name=service_name, - service_role=service_role, - span_prefix=span_prefix, - ), - segment_registry=segment_registry, - ) - runtime.configure() - app.state.policyengine_observability = runtime - set_observability_runtime(runtime) - FastAPIObservabilityAdapter( - runtime, - static_attributes=static_attributes, - ).instrument_app(app) - return runtime + def _end( + self, + *, + status_code: int | None, + error: BaseException | None = None, + ) -> None: + try: + self.runtime.end_request(status_code=status_code, error=error) + except Exception as exc: + self.runtime.diagnostics.report("fastapi.request_finish", exc) def _headers_from_scope(scope: dict[str, Any]) -> dict[str, str]: headers: dict[str, str] = {} - for key, value in scope.get("headers") or []: + for raw_key, raw_value in scope.get("headers") or []: try: - header_key = key.decode("latin-1").lower() - header_value = value.decode("latin-1") - except BaseException: + key = raw_key.decode("latin-1").lower() + value = raw_value.decode("latin-1") + except (AttributeError, UnicodeDecodeError): continue - if header_key in headers: - headers[header_key] = f"{headers[header_key]},{header_value}" - else: - headers[header_key] = header_value + headers[key] = f"{headers[key]},{value}" if key in headers else value return headers def _route_from_scope(scope: dict[str, Any]) -> str | None: - route = scope.get("route") - route_path = getattr(route, "path", None) - if route_path: - return str(route_path) - return None - - -def _endpoint_from_scope(scope: dict[str, Any]) -> str | None: - endpoint = scope.get("endpoint") - if endpoint is None: - return None - endpoint_name = getattr(endpoint, "__name__", None) - return endpoint_name or str(endpoint) - - -def _query_keys(scope: dict[str, Any]) -> list[str]: - query_string = scope.get("query_string") or b"" - try: - decoded = query_string.decode("latin-1") - except AttributeError: - decoded = str(query_string) - return sorted(parse_qs(decoded, keep_blank_values=True).keys()) + path = getattr(scope.get("route"), "path", None) + return str(path) if path else None -def _merge_response_headers( - existing_headers: list[tuple[bytes, bytes]], - headers: dict[str, str], +def _merge_headers( + existing: list[tuple[bytes, bytes]], values: dict[str, str] ) -> list[tuple[bytes, bytes]]: - response_header_names = {key.lower().encode("latin-1") for key in headers} + replacements = {key.lower().encode("latin-1") for key in values} merged = [ (key, value) - for key, value in existing_headers - if key.lower() not in response_header_names + for key, value in existing + if key.lower() not in replacements ] merged.extend( - ( - key.encode("latin-1"), - value.encode("latin-1"), - ) - for key, value in headers.items() - if key in {REQUEST_ID_HEADER, TRACEPARENT_HEADER} + (key.encode("latin-1"), value.encode("latin-1")) + for key, value in values.items() ) return merged - - -def _split_forwarded_for(value: str | None) -> list[str]: - if not value: - return [] - return [part.strip() for part in value.split(",") if part.strip()] - - -def _int_header(value: str | None) -> int | None: - if value is None: - return None - try: - return int(value) - except ValueError: - return None diff --git a/policyengine_observability/adapters/flask.py b/policyengine_observability/adapters/flask.py index 5786392..7801da4 100644 --- a/policyengine_observability/adapters/flask.py +++ b/policyengine_observability/adapters/flask.py @@ -1,132 +1,61 @@ from __future__ import annotations -import uuid from typing import Any -from ..config import ObservabilityConfig -from ..context import RequestObservabilityContext -from ..runtime import ( - OBSERVABILITY_INTERNAL_DISPATCH_HEADER, - REQUEST_ID_HEADER, - ObservabilityRuntime, - set_observability_runtime, -) +from ..runtime import ObservabilityRuntime +_EXTENSION_KEY = "policyengine_observability" -class FlaskObservabilityAdapter: - def __init__(self, runtime: ObservabilityRuntime) -> None: - self.runtime = runtime - def instrument_app(self, app: Any) -> None: - if not self.runtime.enabled: - return - if app.extensions.get("policyengine_observability_adapter"): - return - app.extensions["policyengine_observability_adapter"] = self - - @app.before_request - def _start_observed_request() -> None: - self.start_request() +def instrument_flask( + app: Any, runtime: ObservabilityRuntime +) -> ObservabilityRuntime: + """Install one request lifecycle integration on a Flask application.""" - @app.after_request - def _finish_observed_request(response): - headers = self.runtime.finish_request(response.status_code) - for key, value in headers.items(): - response.headers[key] = value - return response + existing = app.extensions.get(_EXTENSION_KEY) + if isinstance(existing, ObservabilityRuntime): + return existing - @app.teardown_request - def _emit_observed_request(exc) -> None: - self.runtime.teardown_request(exc) + app.extensions[_EXTENSION_KEY] = runtime - def start_request(self) -> None: + @app.before_request + def _begin_observed_request() -> None: try: from flask import request - route = request.url_rule.rule if request.url_rule else request.path - request_id = request.headers.get(REQUEST_ID_HEADER) or str( - uuid.uuid4() + route = ( + request.url_rule.rule if request.url_rule else "" ) - context = RequestObservabilityContext( - config=self.runtime.config, - request_id=request_id, + runtime.begin_request( + headers=dict(request.headers), method=request.method, route=route, - path=request.path, - endpoint=request.endpoint, - query_keys=sorted(request.args.keys()), - content_length_bytes=request.content_length, - inbound=self._inbound_metadata(request), - internal_dispatch=( - request.headers.get(OBSERVABILITY_INTERNAL_DISPATCH_HEADER) - == "1" - ), ) - self.runtime.begin_request(context, carrier=request.headers) - except BaseException as exc: - self.runtime.log_observability_failure("flask.before_request", exc) - - def _inbound_metadata(self, request) -> dict: - forwarded_for = _split_forwarded_for( - request.headers.get("X-Forwarded-For") - ) - x_real_ip = request.headers.get("X-Real-IP") - remote_addr = request.remote_addr - client_ip = None - ip_source = None - if forwarded_for: - client_ip = forwarded_for[0] - ip_source = "x_forwarded_for" - elif x_real_ip: - client_ip = x_real_ip - ip_source = "x_real_ip" - elif remote_addr: - client_ip = remote_addr - ip_source = "remote_addr" - metadata = { - "ip_source": ip_source, - "user_agent": request.headers.get("User-Agent"), - "origin": request.headers.get("Origin"), - "referer": request.headers.get("Referer"), - "host": request.host, - "content_length_bytes": request.content_length, - } - if self.runtime.config.log_raw_ip: - metadata["client_ip"] = client_ip - metadata["forwarded_for"] = forwarded_for - metadata["x_real_ip"] = x_real_ip - return metadata + except Exception as exc: + runtime.diagnostics.report("flask.request_begin", exc) + @app.after_request + def _finish_observed_request(response: Any) -> Any: + try: + from flask import request -def init_flask_observability( - app: Any, - *, - config: ObservabilityConfig | None = None, - runtime: ObservabilityRuntime | None = None, - service_name: str, - service_role: str = "api", - span_prefix: str | None = None, - segment_registry=None, -) -> ObservabilityRuntime: - if app.extensions.get("policyengine_observability"): - return app.extensions["policyengine_observability"] - runtime = runtime or ObservabilityRuntime( - config - or ObservabilityConfig.from_env( - service_name=service_name, - service_role=service_role, - span_prefix=span_prefix, - ), - segment_registry=segment_registry, - ) - runtime.configure() - app.extensions["policyengine_observability"] = runtime - set_observability_runtime(runtime) - FlaskObservabilityAdapter(runtime).instrument_app(app) - return runtime + if request.url_rule is not None: + runtime.update_request_route(request.url_rule.rule) + for key, value in runtime.response_headers().items(): + response.headers[key] = value + runtime.end_request(status_code=response.status_code) + except Exception as exc: + runtime.diagnostics.report("flask.request_finish", exc) + return response + @app.teardown_request + def _close_observed_request(error: BaseException | None) -> None: + try: + runtime.end_request( + status_code=500 if error is not None else None, + error=error, + ) + except Exception as exc: + runtime.diagnostics.report("flask.request_teardown", exc) -def _split_forwarded_for(value: str | None) -> list[str]: - if not value: - return [] - return [part.strip() for part in value.split(",") if part.strip()] + return runtime diff --git a/policyengine_observability/config.py b/policyengine_observability/config.py index 5d58448..6a6afa3 100644 --- a/policyengine_observability/config.py +++ b/policyengine_observability/config.py @@ -1,326 +1,334 @@ from __future__ import annotations -import logging import os -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from typing import Any - -# Defaults for the generic queued-transport knobs; the queued destination -# imports these so a constructor call and an env-configured build can -# never disagree about what "default" means. -DEFAULT_LOG_QUEUE_MAXSIZE = 1000 -DEFAULT_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS = 2.0 - -DEFAULT_METRIC_ATTRIBUTE_KEYS = ( - "service.name", - "service.role", - "deployment.environment", - "operation", - "flavor", - "route", - "method", - "endpoint", - "status_code", - "country_id", - "backend", - "requested_version", - "resolved_channel", - "auth_result", - "segment", - "event", - "error_type", - "model", - "tool", - "stop_reason", - "iteration", - "provider", +from dataclasses import dataclass, field +from typing import Literal, cast + +Platform = Literal["google_cloud_run", "modal", "local", "other"] +OTLPProtocol = Literal["grpc", "http/protobuf"] +ProviderMode = Literal["owned", "external"] + +DEFAULT_APPLICATION_ATTRIBUTE_KEYS = frozenset( + { + "auth_result", + "backend", + "country_id", + "job_type", + "model_version", + "requested_version", + "resolved_channel", + "simulation_year", + } ) +DEFAULT_DISPATCH_ATTRIBUTE_KEYS = frozenset( + { + "job_id", + "run_id", + "simulation_id", + } +) -def bool_from_env(name: str, default: bool) -> bool: - raw_value = os.getenv(name) - if raw_value is None: - return default - return raw_value.strip().lower() not in {"0", "false", "no", "off"} +DEFAULT_METRIC_ATTRIBUTE_KEYS = frozenset( + { + "service.name", + "service.role", + "deployment.environment.name", + "cloud.platform", + "http.route", + "http.request.method", + "http.response.status_code_class", + "operation.name", + "operation.kind", + "outcome", + } +) -def csv_from_env(name: str) -> tuple[str, ...]: - raw_value = os.getenv(name) - if raw_value is None: - return () - return tuple(part.strip() for part in raw_value.split(",") if part.strip()) +@dataclass(frozen=True, slots=True) +class ServiceIdentity: + name: str + namespace: str + version: str + role: str -def float_from_env(name: str, default: float) -> float: - raw_value = os.getenv(name) - if raw_value is None: - return default - try: - return float(raw_value) - except ValueError: - return default +@dataclass(frozen=True, slots=True) +class DeploymentIdentity: + environment: str + platform: Platform + region: str | None = None + instance_id: str | None = None -def int_from_env(name: str, default: int) -> int: - raw_value = os.getenv(name) - if raw_value is None: - return default - try: - return int(raw_value) - except ValueError: - return default +@dataclass(frozen=True, slots=True) +class GoogleCloudLoggingConfig: + project_id: str + log_name: str = "policyengine-api-v1-modal" + queue_capacity: int = 1_000 + batch_size: int = 1 + write_timeout_seconds: float = 5.0 + close_timeout_seconds: float = 2.0 -def default_environment() -> str: - return ( - os.getenv("OBSERVABILITY_ENVIRONMENT") - or os.getenv("DEPLOYMENT_ENVIRONMENT") - or os.getenv("APP_ENV") - or os.getenv("ENVIRONMENT") - or "development" - ) +@dataclass(frozen=True, slots=True) +class LoggingConfig: + stdout_enabled: bool = True + remote: GoogleCloudLoggingConfig | None = None + capture_standard_library: bool = False + replace_existing_handlers: bool = False + minimum_severity: int = 20 -# A log profile is a named preset expanding to generic routing primitives -# (a destination-name tuple and a stdout-formatter name). Presets may name -# strategies; the expansion mechanism knows nothing about any backend. -LOG_PROFILE_PRESETS: dict[str, tuple[tuple[str, ...], str]] = { - # Platforms whose logging agent ingests stdout (Cloud Run, GKE): - # agent-native stdout only, fully synchronous, zero threads. - "gcp-agent": (("stdout",), "google"), - # Platforms with no ingesting agent (Modal): plain stdout as the - # durable record plus queued direct Cloud Logging writes. - "gcp-direct": (("stdout", "google_cloud_logging"), "plain"), - # Local development and the kill switch: plain stdout, zero threads. - "plain-sync": (("stdout",), "plain"), -} - - -def _detect_log_profile() -> str | None: - platform = (os.getenv("OBSERVABILITY_PLATFORM") or "").strip().lower() - if platform == "google_cloud_run": - return "gcp-agent" - if platform == "modal": - return "gcp-direct" - if os.getenv("K_SERVICE"): - return "gcp-agent" - if os.getenv("MODAL_ENVIRONMENT") or os.getenv("MODAL_TASK_ID"): - return "gcp-direct" - return None - - -def _missing_strategy_requirements( - destination_names: Sequence[str], - resolved_config: Mapping[str, Any], -) -> list[tuple[str, str]]: - """(destination, config field) pairs a preset needs but lacks. - - Strategies declare their requirements at registration - (``register_destination(required_config=...)``); this check knows - nothing about any backend. - """ - # Imported lazily: the destinations package imports this module, so - # a module-level import here would be circular. By the time a config - # is resolved the package (and its strategy registrations) is loaded. - from .destinations.registry import destination_strategy - - missing: list[tuple[str, str]] = [] - for name in destination_names: - strategy = destination_strategy(name) - if strategy is None: - continue - for field in strategy.required_config: - if not resolved_config.get(field): - missing.append((name, field)) - return missing +@dataclass(frozen=True, slots=True) +class OTelConfig: + enabled: bool = True + traces_enabled: bool = True + metrics_enabled: bool = True + endpoint: str | None = None + protocol: OTLPProtocol = "grpc" + headers: tuple[tuple[str, str], ...] = () + google_audience: str | None = None + provider_mode: ProviderMode = "owned" + sampling_ratio: float = 1.0 + span_queue_capacity: int = 2_048 + span_batch_size: int = 512 + span_schedule_delay_seconds: float = 5.0 + export_timeout_seconds: float = 5.0 + metric_export_interval_seconds: float = 60.0 -def _resolve_log_profile( - raw_profile: str, - *, - resolved_config: Mapping[str, Any], -) -> tuple[str, tuple[tuple[str, ...], str] | None, list[str]]: - """Resolve a profile name to (name, preset-or-None, warnings). - - ``auto`` without a recognized platform marker resolves to no preset, - so caller-supplied defaults keep applying. ``resolved_config`` - carries the already-resolved config values that registered - strategies may declare as requirements. - """ - warnings: list[str] = [] - # Canonical profile names are hyphenated; accept the same case, - # whitespace, and hyphen/underscore variance as destination and - # formatter names. - profile = raw_profile.strip().lower().replace("_", "-") - if profile == "auto": - detected = _detect_log_profile() - if detected is None: - return "auto", None, warnings - profile = detected - preset = LOG_PROFILE_PRESETS.get(profile) - if preset is None: - warnings.append( - f"Unknown OBSERVABILITY_LOG_PROFILE {raw_profile!r}; " - "using plain-sync." - ) - profile = "plain-sync" - preset = LOG_PROFILE_PRESETS[profile] - missing = _missing_strategy_requirements(preset[0], resolved_config) - if missing: - requirements = ", ".join( - f"{field} (destination {name})" for name, field in missing - ) - warnings.append( - f"Log profile {profile} requires {requirements}; using plain-sync." - ) - profile = "plain-sync" - preset = LOG_PROFILE_PRESETS[profile] - return profile, preset, warnings +@dataclass(frozen=True, slots=True) +class TelemetryLimits: + max_attributes: int = 32 + max_string_length: int = 1_024 + max_error_message_length: int = 2_048 + max_stack_length: int = 16_384 + shutdown_timeout_seconds: float = 3.0 + async_parent_max_age_seconds: float = 300.0 -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class ObservabilityConfig: - service_name: str = "policyengine-service" - service_role: str = "api" - environment: str = "development" - enabled: bool = True - request_logs_enabled: bool = True - log_raw_ip: bool = True - log_level: int = logging.INFO - otel_enabled: bool = True - otlp_endpoint: str | None = None - otlp_protocol: str = "grpc" - span_prefix: str | None = None - tracer_name: str | None = None - meter_name: str | None = None - shutdown_timeout_seconds: float = 3.0 - instrument_fastapi: bool = False - instrument_httpx: bool = False - metric_attribute_keys: tuple[str, ...] = DEFAULT_METRIC_ATTRIBUTE_KEYS - log_destinations: tuple[str, ...] = ("stdout",) - google_cloud_project: str | None = None - google_cloud_log_name: str = "policyengine-observability" - stdout_format: str = "plain" - log_queue_maxsize: int = DEFAULT_LOG_QUEUE_MAXSIZE - log_queue_close_timeout_seconds: float = ( - DEFAULT_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS + service: ServiceIdentity + deployment: DeploymentIdentity + google_cloud_project_id: str | None = None + logging: LoggingConfig = field(default_factory=LoggingConfig) + otel: OTelConfig = field(default_factory=OTelConfig) + limits: TelemetryLimits = field(default_factory=TelemetryLimits) + application_attribute_keys: frozenset[str] = ( + DEFAULT_APPLICATION_ATTRIBUTE_KEYS ) - log_profile: str = "auto" - config_warnings: tuple[str, ...] = () + dispatch_attribute_keys: frozenset[str] = DEFAULT_DISPATCH_ATTRIBUTE_KEYS + metric_attribute_keys: frozenset[str] = DEFAULT_METRIC_ATTRIBUTE_KEYS + sensitive_values: tuple[str, ...] = () @classmethod def from_env( cls, *, - service_name: str, - service_role: str = "api", - enabled_default: bool = True, - span_prefix: str | None = None, - instrument_fastapi: bool = False, - instrument_httpx: bool = False, - metric_attribute_keys: Sequence[str] | None = None, - extra_metric_attribute_keys: Sequence[str] = (), - default_log_destinations: Sequence[str] = ("stdout",), + service: ServiceIdentity, + deployment: DeploymentIdentity, + logging: LoggingConfig | None = None, + google_cloud_project_id: str | None = None, + limits: TelemetryLimits | None = None, + application_attribute_keys: frozenset[str] | None = None, + dispatch_attribute_keys: frozenset[str] | None = None, + metric_attribute_keys: frozenset[str] | None = None, + sensitive_values: tuple[str, ...] = (), ) -> ObservabilityConfig: - level_name = os.getenv("OBSERVABILITY_LOG_LEVEL", "INFO").upper() - log_level = getattr(logging, level_name, logging.INFO) - otlp_protocol = ( - os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL") - or os.getenv("OBSERVABILITY_OTLP_PROTOCOL") - or cls.otlp_protocol - ) - env_metric_keys = csv_from_env("OBSERVABILITY_METRIC_ATTRIBUTE_KEYS") - env_extra_metric_keys = csv_from_env( - "OBSERVABILITY_EXTRA_METRIC_ATTRIBUTE_KEYS" - ) - env_log_destinations = csv_from_env("OBSERVABILITY_LOG_DESTINATIONS") - resolved_metric_keys = _dedupe( - env_metric_keys - or metric_attribute_keys - or DEFAULT_METRIC_ATTRIBUTE_KEYS, - (*extra_metric_attribute_keys, *env_extra_metric_keys), - ) - google_cloud_project = ( - os.getenv("OBSERVABILITY_GOOGLE_CLOUD_PROJECT") - or os.getenv("GOOGLE_CLOUD_PROJECT") - or os.getenv("GCP_PROJECT") - or os.getenv("GCLOUD_PROJECT") - or None - ) - log_profile, preset, profile_warnings = _resolve_log_profile( - os.getenv("OBSERVABILITY_LOG_PROFILE") or cls.log_profile, - # The values strategies may declare via required_config; - # extend as future fields become requirement candidates. - resolved_config={"google_cloud_project": google_cloud_project}, - ) - profile_destinations, profile_stdout_format = preset or (None, None) - # Explicit granular env vars override the profile's expansion; - # the profile overrides caller-supplied defaults. - resolved_log_destinations = _dedupe( - env_log_destinations - or profile_destinations - or default_log_destinations - ) - resolved_stdout_format = ( - os.getenv("OBSERVABILITY_STDOUT_FORMAT") - or profile_stdout_format - or cls.stdout_format - ) + """Read standard OTel transport settings with explicit identity. + + Service identity, deployment identity, and log routing are never + inferred from ambient platform or Google Cloud variables. + """ + + endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") + protocol = os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").strip() + if protocol not in {"grpc", "http/protobuf"}: + protocol = "grpc" + return cls( - service_name=os.getenv("OBSERVABILITY_SERVICE_NAME") - or os.getenv("OTEL_SERVICE_NAME") - or service_name, - service_role=service_role, - environment=default_environment(), - enabled=bool_from_env("OBSERVABILITY_ENABLED", enabled_default), - request_logs_enabled=bool_from_env( - "OBSERVABILITY_REQUEST_LOGS_ENABLED", - True, - ), - log_raw_ip=bool_from_env("OBSERVABILITY_LOG_RAW_IP", True), - log_level=log_level, - otel_enabled=bool_from_env("OTEL_ENABLED", cls.otel_enabled), - otlp_endpoint=os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") or None, - otlp_protocol=otlp_protocol, - span_prefix=span_prefix, - tracer_name=os.getenv("OBSERVABILITY_TRACER_NAME"), - meter_name=os.getenv("OBSERVABILITY_METER_NAME"), - shutdown_timeout_seconds=float_from_env( - "OBSERVABILITY_SHUTDOWN_TIMEOUT_SECONDS", - cls.shutdown_timeout_seconds, + service=service, + deployment=deployment, + google_cloud_project_id=google_cloud_project_id, + logging=logging or LoggingConfig(), + otel=OTelConfig( + enabled=not _env_bool("OTEL_SDK_DISABLED", False), + traces_enabled=os.getenv("OTEL_TRACES_EXPORTER", "otlp") + != "none", + metrics_enabled=os.getenv("OTEL_METRICS_EXPORTER", "otlp") + != "none", + endpoint=endpoint, + protocol=cast(OTLPProtocol, protocol), + headers=_parse_headers( + os.getenv("OTEL_EXPORTER_OTLP_HEADERS", "") + ), + google_audience=os.getenv("POLICYENGINE_OTEL_GOOGLE_AUDIENCE"), + provider_mode=_provider_mode( + os.getenv("POLICYENGINE_OTEL_PROVIDER_MODE", "owned") + ), + sampling_ratio=_bounded_float( + os.getenv("OTEL_TRACES_SAMPLER_ARG"), + default=1.0, + minimum=0.0, + maximum=1.0, + ), + span_queue_capacity=_bounded_int( + os.getenv("OTEL_BSP_MAX_QUEUE_SIZE"), + default=2_048, + minimum=1, + maximum=100_000, + ), + span_batch_size=_bounded_int( + os.getenv("OTEL_BSP_MAX_EXPORT_BATCH_SIZE"), + default=512, + minimum=1, + maximum=10_000, + ), + span_schedule_delay_seconds=( + _bounded_float( + os.getenv("OTEL_BSP_SCHEDULE_DELAY"), + default=5_000.0, + minimum=1.0, + maximum=60_000.0, + ) + / 1_000 + ), + export_timeout_seconds=( + _bounded_float( + os.getenv("OTEL_BSP_EXPORT_TIMEOUT"), + default=5_000.0, + minimum=1.0, + maximum=60_000.0, + ) + / 1_000 + ), + metric_export_interval_seconds=( + _bounded_float( + os.getenv("OTEL_METRIC_EXPORT_INTERVAL"), + default=60_000.0, + minimum=1_000.0, + maximum=3_600_000.0, + ) + / 1_000 + ), ), - instrument_fastapi=bool_from_env( - "OBSERVABILITY_INSTRUMENT_FASTAPI", - instrument_fastapi, + limits=limits or TelemetryLimits(), + application_attribute_keys=( + application_attribute_keys + if application_attribute_keys is not None + else DEFAULT_APPLICATION_ATTRIBUTE_KEYS ), - instrument_httpx=bool_from_env( - "OBSERVABILITY_INSTRUMENT_HTTPX", - instrument_httpx, + dispatch_attribute_keys=( + dispatch_attribute_keys + if dispatch_attribute_keys is not None + else DEFAULT_DISPATCH_ATTRIBUTE_KEYS ), - metric_attribute_keys=resolved_metric_keys, - log_destinations=resolved_log_destinations, - google_cloud_project=google_cloud_project, - google_cloud_log_name=( - os.getenv("OBSERVABILITY_GOOGLE_CLOUD_LOG_NAME") - or cls.google_cloud_log_name + metric_attribute_keys=( + metric_attribute_keys + if metric_attribute_keys is not None + else DEFAULT_METRIC_ATTRIBUTE_KEYS ), - stdout_format=resolved_stdout_format, - log_queue_maxsize=int_from_env( - "OBSERVABILITY_LOG_QUEUE_MAXSIZE", - cls.log_queue_maxsize, - ), - log_queue_close_timeout_seconds=float_from_env( - "OBSERVABILITY_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS", - cls.log_queue_close_timeout_seconds, - ), - log_profile=log_profile, - config_warnings=tuple(profile_warnings), + sensitive_values=sensitive_values, + ) + + def diagnostics(self) -> tuple[str, ...]: + messages: list[str] = [] + identity_values = { + "service.name": self.service.name, + "service.namespace": self.service.namespace, + "service.version": self.service.version, + "service.role": self.service.role, + "deployment.environment.name": self.deployment.environment, + "cloud.platform": self.deployment.platform, + } + for key, value in identity_values.items(): + if not str(value).strip(): + messages.append(f"Missing explicit runtime identity: {key}") + + remote = self.logging.remote + if remote is not None and ( + not remote.project_id.strip() or not remote.log_name.strip() + ): + messages.append( + "Remote logging is disabled because its project or log name " + "is empty." + ) + if remote is not None and self.deployment.platform != "modal": + messages.append( + "Direct Cloud Logging is disabled outside Modal; use platform " + "standard-output ingestion." + ) + if self.otel.enabled and not self.otel.endpoint: + messages.append( + "Remote OTel export is disabled because no OTLP endpoint is " + "configured." + ) + return tuple(messages) + + @property + def identity_complete(self) -> bool: + return not any( + not str(value).strip() + for value in ( + self.service.name, + self.service.namespace, + self.service.version, + self.service.role, + self.deployment.environment, + self.deployment.platform, + ) ) -def _dedupe( - base: Sequence[str], - extra: Sequence[str] = (), -) -> tuple[str, ...]: - return tuple(dict.fromkeys((*base, *extra))) +def _env_bool(name: str, default: bool) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _provider_mode(value: str) -> ProviderMode: + return "external" if value.strip().lower() == "external" else "owned" + + +def _parse_headers(value: str) -> tuple[tuple[str, str], ...]: + headers: list[tuple[str, str]] = [] + for item in value.split(","): + if not item.strip() or "=" not in item: + continue + key, header_value = item.split("=", 1) + if key.strip(): + headers.append((key.strip(), header_value.strip())) + return tuple(headers) + + +def _bounded_float( + value: str | None, + *, + default: float, + minimum: float, + maximum: float, +) -> float: + try: + parsed = float(value) if value is not None else default + except (TypeError, ValueError): + return default + if parsed != parsed: + return default + return min(max(parsed, minimum), maximum) + + +def _bounded_int( + value: str | None, + *, + default: int, + minimum: int, + maximum: int, +) -> int: + try: + parsed = int(value) if value is not None else default + except (TypeError, ValueError): + return default + return min(max(parsed, minimum), maximum) diff --git a/policyengine_observability/context.py b/policyengine_observability/context.py deleted file mode 100644 index bb2d464..0000000 --- a/policyengine_observability/context.py +++ /dev/null @@ -1,270 +0,0 @@ -from __future__ import annotations - -import time -from dataclasses import dataclass, field -from datetime import UTC, datetime -from typing import Any - -from .config import ObservabilityConfig - - -@dataclass -class ErrorRecord: - type: str - message: str - handled: bool - stack: str | None = None - - def as_dict(self) -> dict[str, Any]: - return { - "type": self.type, - "message": self.message, - "handled": self.handled, - "stack": self.stack, - } - - -@dataclass -class SegmentTimingNode: - sequence: int - name: str - attrs: dict[str, Any] = field(default_factory=dict) - duration_ms: float | None = None - children: list[SegmentTimingNode] = field(default_factory=list) - - def as_dict(self) -> dict[str, Any]: - record: dict[str, Any] = { - "sequence": self.sequence, - "name": self.name, - } - if self.attrs: - record["attrs"] = dict(self.attrs) - if self.duration_ms is not None: - record["duration_ms"] = round(self.duration_ms, 3) - if self.children: - record["children"] = [child.as_dict() for child in self.children] - return record - - -@dataclass -class OperationObservabilityContext: - config: ObservabilityConfig - name: str - flavor: str | None = None - attributes: dict[str, Any] = field(default_factory=dict) - timings_ms: dict[str, float] = field(default_factory=dict) - timing_counts: dict[str, int] = field(default_factory=dict) - segment_tree: list[SegmentTimingNode] = field(default_factory=list) - segment_sequence: list[int] = field(default_factory=lambda: [0]) - emit_log: bool = True - record_metric: bool = True - started_at: float = field(default_factory=time.perf_counter) - created_at: datetime = field(default_factory=lambda: datetime.now(UTC)) - error: ErrorRecord | None = None - emitted: bool = False - metric_recorded: bool = False - span_handle: Any = None - context_token: Any = None - - def set_attribute(self, key: str, value: Any) -> None: - if value is None: - return - if hasattr(value, "value"): - value = value.value - self.attributes[key] = value - - def duration_seconds(self) -> float: - return time.perf_counter() - self.started_at - - def metric_attributes(self, **extra: Any) -> dict[str, str]: - attrs: dict[str, Any] = { - "service.name": self.config.service_name, - "service.role": self.config.service_role, - "deployment.environment": self.config.environment, - "operation": self.name, - "flavor": self.flavor, - } - for key in self.config.metric_attribute_keys: - if key in self.attributes: - attrs[key] = self.attributes[key] - attrs.update(extra) - return _metric_attrs(attrs, self.config.metric_attribute_keys) - - def span_attributes(self, **extra: Any) -> dict[str, Any]: - attrs: dict[str, Any] = { - "service.name": self.config.service_name, - "service.role": self.config.service_role, - "deployment.environment": self.config.environment, - "policyengine.operation": self.name, - "policyengine.flavor": self.flavor, - } - attrs.update( - { - f"policyengine.{key}": value - for key, value in self.attributes.items() - if value is not None - } - ) - attrs.update(extra) - return { - key: value for key, value in attrs.items() if value is not None - } - - def as_log_record( - self, - *, - trace_id: str | None, - span_id: str | None, - ) -> dict[str, Any]: - event = "operation_failed" if self.error else "operation_completed" - return { - **self.attributes, - "schema_version": "policyengine.observability.operation.v1", - "event": event, - "service_name": self.config.service_name, - "service_role": self.config.service_role, - "environment": self.config.environment, - "created_at": self.created_at.isoformat(), - "operation": self.name, - "flavor": self.flavor, - "trace_id": trace_id, - "span_id": span_id, - "duration_ms": round(self.duration_seconds() * 1000, 3), - "timings_ms": dict(self.timings_ms), - "timing_counts": dict(self.timing_counts), - "segment_tree": [node.as_dict() for node in self.segment_tree], - "error": self.error.as_dict() if self.error else None, - } - - -@dataclass -class RequestObservabilityContext: - config: ObservabilityConfig - request_id: str - method: str - route: str - path: str - endpoint: str | None - query_keys: list[str] - content_length_bytes: int | None - inbound: dict[str, Any] - internal_dispatch: bool = False - started_at: float = field(default_factory=time.perf_counter) - created_at: datetime = field(default_factory=lambda: datetime.now(UTC)) - attributes: dict[str, Any] = field(default_factory=dict) - timings_ms: dict[str, float] = field(default_factory=dict) - timing_counts: dict[str, int] = field(default_factory=dict) - segment_tree: list[SegmentTimingNode] = field(default_factory=list) - segment_sequence: list[int] = field(default_factory=lambda: [0]) - status_code: int | None = None - error: ErrorRecord | None = None - emitted: bool = False - request_metric_recorded: bool = False - active_closed: bool = False - span_closed: bool = False - server_span_cm: Any = None - server_span: Any = None - context_token: Any = None - operation_context: OperationObservabilityContext | None = None - operation_token: Any = None - - def set_attribute(self, key: str, value: Any) -> None: - if value is None: - return - if hasattr(value, "value"): - value = value.value - self.attributes[key] = value - - def duration_seconds(self) -> float: - return time.perf_counter() - self.started_at - - def metric_attributes(self, **extra: Any) -> dict[str, str]: - attrs: dict[str, Any] = { - "service.name": self.config.service_name, - "service.role": self.config.service_role, - "deployment.environment": self.config.environment, - "route": self.route, - "method": self.method, - "endpoint": self.endpoint, - } - if self.status_code is not None: - attrs["status_code"] = str(self.status_code) - for key in self.config.metric_attribute_keys: - if key in self.attributes: - attrs[key] = self.attributes[key] - attrs.update(extra) - return _metric_attrs(attrs, self.config.metric_attribute_keys) - - def span_attributes(self, **extra: Any) -> dict[str, Any]: - attrs: dict[str, Any] = { - "service.name": self.config.service_name, - "service.role": self.config.service_role, - "deployment.environment": self.config.environment, - "http.request.method": self.method, - "http.route": self.route, - "url.path": self.path, - "policyengine.endpoint": self.endpoint, - "policyengine.request_id": self.request_id, - } - if self.status_code is not None: - attrs["http.response.status_code"] = self.status_code - for key in ( - "country_id", - "backend", - "requested_version", - "resolved_channel", - "auth_result", - ): - if key in self.attributes: - attrs[f"policyengine.{key}"] = self.attributes[key] - attrs.update(extra) - return { - key: value for key, value in attrs.items() if value is not None - } - - def as_log_record( - self, - *, - trace_id: str | None, - span_id: str | None, - ) -> dict[str, Any]: - event = ( - "http_request_failed" if self.error else "http_request_completed" - ) - status_code = self.status_code or (500 if self.error else None) - return { - **self.inbound, - **self.attributes, - "schema_version": "policyengine.observability.request.v1", - "event": event, - "service_name": self.config.service_name, - "service_role": self.config.service_role, - "environment": self.config.environment, - "created_at": self.created_at.isoformat(), - "request_id": self.request_id, - "trace_id": trace_id, - "span_id": span_id, - "method": self.method, - "route": self.route, - "path": self.path, - "query_keys": self.query_keys, - "endpoint": self.endpoint, - "status_code": status_code, - "duration_ms": round(self.duration_seconds() * 1000, 3), - "timings_ms": dict(self.timings_ms), - "timing_counts": dict(self.timing_counts), - "segment_tree": [node.as_dict() for node in self.segment_tree], - "error": self.error.as_dict() if self.error else None, - } - - -def _metric_attrs( - attrs: dict[str, Any], - metric_attribute_keys: tuple[str, ...], -) -> dict[str, str]: - result: dict[str, str] = {} - for key in metric_attribute_keys: - value = attrs.get(key) - if value is not None: - result[key] = str(value) - return result diff --git a/policyengine_observability/delivery.py b/policyengine_observability/delivery.py new file mode 100644 index 0000000..4053a05 --- /dev/null +++ b/policyengine_observability/delivery.py @@ -0,0 +1,272 @@ +from __future__ import annotations + +import functools +import json +import queue +import sys +import threading +from collections.abc import Callable +from typing import Any, Protocol, TextIO + +from .config import GoogleCloudLoggingConfig, ObservabilityConfig +from .diagnostics import Diagnostics + + +class RecordWriter(Protocol): + def write(self, record: dict[str, Any]) -> None: ... + + def write_many(self, records: list[dict[str, Any]]) -> None: ... + + def close(self) -> None: ... + + +class DeliveryManager: + def __init__( + self, + config: ObservabilityConfig, + diagnostics: Diagnostics, + *, + stdout: TextIO | None = None, + writer_factory: Callable[[GoogleCloudLoggingConfig], RecordWriter] + | None = None, + ) -> None: + self.config = config + self.diagnostics = diagnostics + self._stdout = stdout or sys.stdout + self._writer_factory = writer_factory or _GoogleCloudWriter + self._remote: _QueuedWriter | None = None + remote = config.logging.remote + if ( + remote is not None + and config.deployment.platform == "modal" + and config.identity_complete + and remote.project_id.strip() + and remote.log_name.strip() + ): + self._remote = _QueuedWriter( + remote, + diagnostics, + writer_factory=self._writer_factory, + ) + + def emit(self, record: dict[str, Any]) -> None: + if self.config.logging.stdout_enabled: + try: + print( + json.dumps( + record, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ), + file=self._stdout, + flush=True, + ) + except Exception as exc: + self.diagnostics.report("stdout.write", exc) + if self._remote is not None: + self._remote.enqueue(record.copy()) + + def close(self, timeout_seconds: float | None = None) -> None: + if self._remote is None: + return + self._remote.close(timeout_seconds) + + @property + def remote_enabled(self) -> bool: + return self._remote is not None + + @property + def queue_depth(self) -> int: + return self._remote.queue_depth if self._remote else 0 + + +class _QueuedWriter: + _STOP = object() + + def __init__( + self, + config: GoogleCloudLoggingConfig, + diagnostics: Diagnostics, + *, + writer_factory: Callable[[GoogleCloudLoggingConfig], RecordWriter], + ) -> None: + self.config = config + self.diagnostics = diagnostics + self._writer_factory = writer_factory + self._queue: queue.Queue[dict[str, Any] | object] = queue.Queue( + maxsize=max(1, min(config.queue_capacity, 100_000)) + ) + self._closed = threading.Event() + self._writer: RecordWriter | None = None + self._thread = threading.Thread( + target=self._run, + name="policyengine-observability-log-writer", + daemon=True, + ) + self._thread.start() + + def enqueue(self, record: dict[str, Any]) -> None: + if self._closed.is_set(): + self.diagnostics.increment("logs.dropped.closed") + return + try: + self._queue.put_nowait(record) + except queue.Full: + self.diagnostics.increment("logs.dropped.queue_full") + self.diagnostics.report( + "logging.queue_full", + "Remote log queue is full; newest record dropped.", + capacity=self._queue.maxsize, + ) + + @property + def queue_depth(self) -> int: + return self._queue.qsize() + + def close(self, timeout_seconds: float | None = None) -> None: + if self._closed.is_set(): + return + self._closed.set() + try: + self._queue.put_nowait(self._STOP) + except queue.Full: + pass + timeout = ( + self.config.close_timeout_seconds + if timeout_seconds is None + else timeout_seconds + ) + self._thread.join(max(0.0, timeout)) + if self._thread.is_alive(): + self.diagnostics.increment("logs.shutdown_timeout") + self.diagnostics.report( + "logging.shutdown_timeout", + "Remote log worker did not stop before its deadline.", + ) + + def _run(self) -> None: + try: + while True: + try: + item = self._queue.get(timeout=0.2) + except queue.Empty: + if self._closed.is_set(): + return + continue + consumed = [item] + stop_after_batch = item is self._STOP + batch: list[dict[str, Any]] = [] + if isinstance(item, dict): + batch.append(item) + while not stop_after_batch and len(batch) < max( + 1, self.config.batch_size + ): + try: + next_item = self._queue.get_nowait() + except queue.Empty: + break + consumed.append(next_item) + if next_item is self._STOP: + stop_after_batch = True + elif isinstance(next_item, dict): + batch.append(next_item) + try: + if not batch: + return + if self._writer is None: + self._writer = self._writer_factory(self.config) + write_many = getattr(self._writer, "write_many", None) + if callable(write_many): + write_many(batch) + else: + for record in batch: + self._writer.write(record) + except Exception as exc: + self.diagnostics.increment("logs.export_failure") + self.diagnostics.report("logging.export", exc) + finally: + for _consumed_item in consumed: + self._queue.task_done() + if stop_after_batch: + return + finally: + if self._writer is not None: + try: + self._writer.close() + except Exception as exc: + self.diagnostics.report("logging.writer_close", exc) + + +class _GoogleCloudWriter: + """Lazy worker-thread-only Cloud Logging client.""" + + def __init__(self, config: GoogleCloudLoggingConfig) -> None: + from google.cloud import logging_v2 + + from .google_credentials import load_google_credentials + + credentials = load_google_credentials(prefer_workload_identity=True) + self._client = logging_v2.Client( + project=config.project_id, + credentials=credentials, + ) + self._logger = self._client.logger(config.log_name) + self._timeout = max(0.1, min(config.write_timeout_seconds, 60.0)) + self._bound_write_timeout() + logging_v2._instrumentation_emitted = True + + def write(self, record: dict[str, Any]) -> None: + self.write_many([record]) + + def write_many(self, records: list[dict[str, Any]]) -> None: + batch = self._logger.batch() + for record in records: + kwargs = self._entry_kwargs(record) + batch.log_struct(record, **kwargs) + batch.commit() + + def _entry_kwargs(self, record: dict[str, Any]) -> dict[str, Any]: + kwargs: dict[str, Any] = { + "severity": record.get("severity", "DEFAULT"), + } + trace = record.get("logging.googleapis.com/trace") + span_id = record.get("logging.googleapis.com/spanId") + if trace: + kwargs["trace"] = trace + if span_id: + kwargs["span_id"] = span_id + kwargs["trace_sampled"] = bool( + record.get("logging.googleapis.com/trace_sampled", False) + ) + return kwargs + + def close(self) -> None: + close = getattr(self._client, "close", None) + if callable(close): + close() + + def _bound_write_timeout(self) -> None: + api = getattr(self._client, "logging_api", None) + gapic = getattr(api, "_gapic_api", None) + if gapic is None: + return + from google.api_core import exceptions as api_exceptions + from google.api_core.retry import Retry, if_exception_type + + retry = Retry( + initial=0.1, + maximum=1.0, + multiplier=1.3, + timeout=self._timeout, + predicate=if_exception_type( + api_exceptions.DeadlineExceeded, + api_exceptions.InternalServerError, + api_exceptions.ServiceUnavailable, + ), + ) + gapic.write_log_entries = functools.partial( + gapic.write_log_entries, + retry=retry, + timeout=self._timeout, + ) diff --git a/policyengine_observability/destinations/__init__.py b/policyengine_observability/destinations/__init__.py deleted file mode 100644 index e37847b..0000000 --- a/policyengine_observability/destinations/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import annotations - -from .base import LogDestination, normalize_payload -from .google_cloud_logging import GoogleCloudLoggingDestination -from .manager import LogDestinationManager -from .queued import QueuedLogDestination -from .registry import register_destination -from .stdout import StdoutJsonDestination, register_stdout_formatter - -__all__ = [ - "GoogleCloudLoggingDestination", - "LogDestination", - "LogDestinationManager", - "QueuedLogDestination", - "StdoutJsonDestination", - "normalize_payload", - "register_destination", - "register_stdout_formatter", -] diff --git a/policyengine_observability/destinations/base.py b/policyengine_observability/destinations/base.py deleted file mode 100644 index 8c93e09..0000000 --- a/policyengine_observability/destinations/base.py +++ /dev/null @@ -1,121 +0,0 @@ -from __future__ import annotations - -import inspect -import math -from collections.abc import Callable, Mapping, Sequence -from typing import Any, Protocol - - -class LogDestination(Protocol): - name: str - - def emit( - self, - payload: dict[str, Any], - *, - log_type: str, - severity: str, - ) -> None: - """Write one structured observability payload.""" - - -def normalize_name(name: str) -> str: - """Canonical lookup key for registered names. - - Destination, formatter, and profile lookups all forgive case, - surrounding whitespace, and hyphen/underscore variance the same way, - so a spelling that works for one registry works for every registry. - """ - return name.strip().lower().replace("-", "_") - - -def accepts_keyword(func: Callable[..., Any], name: str) -> bool: - """Whether ``func`` can safely be called with keyword ``name``.""" - try: - parameters = inspect.signature(func).parameters - except (TypeError, ValueError): - return False - if name in parameters: - return True - return any( - parameter.kind is inspect.Parameter.VAR_KEYWORD - for parameter in parameters.values() - ) - - -def safe_report( - on_failure: Callable[..., None], - operation: str, - exc: BaseException, - **fields: Any, -) -> None: - """Report through the internal-error channel; never raises.""" - try: - on_failure(operation, exc, **fields) - except Exception: - pass - - -def close_destination( - destination: LogDestination, - *, - on_failure: Callable[..., None], - deadline_seconds: float | None = None, -) -> None: - """Close a destination if it supports closing; never raises. - - ``close`` is duck-typed with one calling convention everywhere: the - deadline is passed only when the signature accepts it, so both - ``close(self)`` and ``close(self, deadline_seconds=None)`` work under - every close path (manager shutdown, reconfigure, queued drain). - """ - close = getattr(destination, "close", None) - if not callable(close): - return - try: - if accepts_keyword(close, "deadline_seconds"): - close(deadline_seconds=deadline_seconds) - else: - close() - except Exception as exc: - safe_report( - on_failure, - "logging.destination_close", - exc, - destination=getattr(destination, "name", None), - ) - - -def clamped(value: Any, *, low: float, high: float, default: float) -> float: - """Coerce a config knob to a finite float within [low, high]. - - Anything unparseable or non-finite falls back to the default, so a - stray env value can never disable or unbound the mechanism it tunes. - """ - try: - number = float(value) - except (TypeError, ValueError): - return default - if not math.isfinite(number): - return default - return min(max(number, low), high) - - -def normalize_payload(value: Any) -> Any: - if value is None or isinstance(value, str | bool | int | float): - return value - if isinstance(value, Mapping): - return { - str(key): normalize_payload(item) for key, item in value.items() - } - if isinstance(value, Sequence) and not isinstance(value, str | bytes): - return [normalize_payload(item) for item in value] - if isinstance(value, bytes): - try: - return value.decode("utf-8") - except UnicodeDecodeError: - return repr(value) - try: - return str(value) - except BaseException: - return f"" diff --git a/policyengine_observability/destinations/google_cloud_logging.py b/policyengine_observability/destinations/google_cloud_logging.py deleted file mode 100644 index 2e1c96e..0000000 --- a/policyengine_observability/destinations/google_cloud_logging.py +++ /dev/null @@ -1,241 +0,0 @@ -from __future__ import annotations - -import functools -from collections.abc import Callable -from datetime import datetime -from typing import Any, Protocol - -from ..config import float_from_env -from .base import clamped, normalize_payload -from .google_credentials import ( - configure_google_application_credentials, - load_google_credentials, -) -from .registry import register_destination -from .stdout import StdoutFormatter, register_stdout_formatter - -DEFAULT_WRITE_TIMEOUT_SECONDS = 10.0 -MIN_WRITE_TIMEOUT_SECONDS = 0.5 -MAX_WRITE_TIMEOUT_SECONDS = 60.0 - -# Structured-JSON keys the Cloud Run/GKE logging agent promotes to -# first-class LogEntry fields when it ingests a stdout line. -GOOGLE_TRACE_KEY = "logging.googleapis.com/trace" -GOOGLE_SPAN_ID_KEY = "logging.googleapis.com/spanId" -GOOGLE_LABELS_KEY = "logging.googleapis.com/labels" - - -class GoogleCloudLogger(Protocol): - def log_struct( - self, - payload: dict[str, Any], - **kwargs: Any, - ) -> None: ... - - -class GoogleCloudLoggingClient(Protocol): - project: str | None - - def logger(self, log_name: str) -> GoogleCloudLogger: ... - - -GoogleCloudLoggingClientFactory = Callable[ - [str | None, object | None], - GoogleCloudLoggingClient, -] - - -class GoogleCloudLoggingDestination: - name = "google_cloud_logging" - - def __init__( - self, - *, - project: str | None, - log_name: str, - client_factory: GoogleCloudLoggingClientFactory | None = None, - write_timeout_seconds: float | None = None, - ) -> None: - self.project = project - self.log_name = log_name - self.write_timeout_seconds = clamped( - write_timeout_seconds, - low=MIN_WRITE_TIMEOUT_SECONDS, - high=MAX_WRITE_TIMEOUT_SECONDS, - default=DEFAULT_WRITE_TIMEOUT_SECONDS, - ) - credentials = load_google_credentials(prefer_workload_identity=True) - if credentials is None: - configure_google_application_credentials() - if client_factory is None: - from google.cloud import logging as cloud_logging - - def client_factory( - project_id: str | None, - credentials: object | None, - ) -> GoogleCloudLoggingClient: - return cloud_logging.Client( - project=project_id, - credentials=credentials, - ) - - self.client = client_factory(project, credentials) - self.project = project or getattr(self.client, "project", None) - self.logger = self.client.logger(log_name) - self._bound_write_timeout() - self._suppress_instrumentation_entry() - - def _suppress_instrumentation_entry(self) -> None: - """Keep the library's diagnostic entry out of the log stream. - - ``log_struct`` prepends a one-time instrumentation diagnostic - entry to the first write per process; observability entries - should be only the records we were asked to write. - """ - try: - from google.cloud import logging_v2 - - logging_v2._instrumentation_emitted = True - except ImportError: # pragma: no cover - google extra has it - pass - - def _bound_write_timeout(self) -> None: - """Bound every write on this destination's own client. - - ``log_struct`` exposes no call options, and the transport default - lets a degraded Logging API hold one write for up to 60 seconds. - The gapic method is the single choke point underneath - ``log_struct``, so rebind it on this client instance with a retry - whose transient-error budget and per-call timeout are both capped - by ``write_timeout_seconds``. When the private handle is absent - (HTTP transport, injected fakes), the library default applies — - acceptable because no write on this destination ever runs on a - request thread. - """ - api = getattr(self.client, "logging_api", None) - gapic = getattr(api, "_gapic_api", None) - if gapic is None: - return - try: - from google.api_core import exceptions as api_exceptions - from google.api_core.retry import Retry, if_exception_type - except ImportError: # pragma: no cover - google extra always has it - return - retry = Retry( - initial=0.1, - maximum=1.0, - multiplier=1.3, - timeout=self.write_timeout_seconds, - predicate=if_exception_type( - api_exceptions.DeadlineExceeded, - api_exceptions.InternalServerError, - api_exceptions.ServiceUnavailable, - ), - ) - gapic.write_log_entries = functools.partial( - gapic.write_log_entries, - retry=retry, - timeout=self.write_timeout_seconds, - ) - - def emit( - self, - payload: dict[str, Any], - *, - log_type: str, - severity: str, - timestamp: datetime | None = None, - ) -> None: - normalized = normalize_payload(payload) - kwargs: dict[str, Any] = { - "severity": severity, - "labels": _labels(normalized, log_type=log_type), - } - if timestamp is not None: - # Supplied by queued transports so delayed writes keep the - # record's event time instead of the delivery time. - kwargs["timestamp"] = timestamp - trace_id = normalized.get("trace_id") - if trace_id and self.project: - kwargs["trace"] = _trace_resource(self.project, trace_id) - span_id = normalized.get("span_id") - if span_id: - kwargs["span_id"] = span_id - self.logger.log_struct(normalized, **kwargs) - - def close(self) -> None: - """Release the owned client's transport, if it supports it.""" - close = getattr(self.client, "close", None) - if callable(close): - close() - - -def _trace_resource(project: str, trace_id: str) -> str: - # The LogEntry trace resource name; the direct write path and the - # agent-native stdout formatter must build it identically for trace - # correlation to work. - return f"projects/{project}/traces/{trace_id}" - - -def _labels(payload: dict[str, Any], *, log_type: str) -> dict[str, str]: - labels = {"log_type": log_type} - for key in ( - "service_name", - "service_role", - "environment", - "schema_version", - ): - value = payload.get(key) - if value is not None: - labels[key] = str(value) - return labels - - -def _google_stdout_formatter_factory(config: Any) -> StdoutFormatter: - """Shape stdout lines with the agent-native Cloud Logging keys. - - Emission stays synchronous, so no ``time`` key is set — the agent's - receive time is the event time. - """ - project = getattr(config, "google_cloud_project", None) - - def format_google( - payload: dict[str, Any], *, log_type: str, severity: str - ) -> dict[str, Any]: - payload["severity"] = str(severity).upper() - payload[GOOGLE_LABELS_KEY] = _labels(payload, log_type=log_type) - trace_id = payload.get("trace_id") - if trace_id and project: - payload[GOOGLE_TRACE_KEY] = _trace_resource(project, trace_id) - span_id = payload.get("span_id") - if span_id: - payload[GOOGLE_SPAN_ID_KEY] = str(span_id) - return payload - - return format_google - - -register_stdout_formatter("google", _google_stdout_formatter_factory) - - -def _google_destination_factory(*, config: Any, **_: Any): - return GoogleCloudLoggingDestination( - project=config.google_cloud_project, - log_name=config.google_cloud_log_name, - # Backend knobs belong to the strategy: parsed here at - # construction (so re-read on restart_observability()), keeping - # per-backend fields off the core config dataclass. - write_timeout_seconds=float_from_env( - "OBSERVABILITY_GOOGLE_WRITE_TIMEOUT_SECONDS", - DEFAULT_WRITE_TIMEOUT_SECONDS, - ), - ) - - -register_destination( - "google_cloud_logging", - _google_destination_factory, - transport="remote", - aliases=("google", "google_cloud"), - required_config=("google_cloud_project",), -) diff --git a/policyengine_observability/destinations/manager.py b/policyengine_observability/destinations/manager.py deleted file mode 100644 index 086e377..0000000 --- a/policyengine_observability/destinations/manager.py +++ /dev/null @@ -1,223 +0,0 @@ -from __future__ import annotations - -import logging -import time -from collections.abc import Callable, Mapping -from typing import Any - -from ..config import ObservabilityConfig -from .base import LogDestination, close_destination -from .queued import QueuedLogDestination -from .registry import destination_strategy -from .stdout import StdoutJsonDestination, build_stdout_destination - -# A destination that fails this many consecutive emits is disabled for the -# rest of the process. Inline destinations emit synchronously on the -# caller's (request) path, so a persistently failing one — e.g. a broken -# serializer — must not keep charging every request; an observability sink -# can never be allowed to degrade the host service. Queued destinations -# never raise from emit (drops are counted internally), so this breaker -# only ever governs inline strategies. -DESTINATION_FAILURE_LIMIT = 3 - - -class LogDestinationManager: - def __init__( - self, - *, - config: ObservabilityConfig, - loggers: Mapping[str, logging.Logger], - serializer: Callable[[dict[str, Any]], str], - on_failure: Callable[..., None], - ) -> None: - self.config = config - self.loggers = loggers - self.serializer = serializer - self.on_failure = on_failure - self.destinations: list[LogDestination] = [] - self.configured = False - self._consecutive_failures: dict[int, int] = {} - - def configure(self) -> None: - # Reconfigure (restart_observability) runs only from - # single-threaded lifecycle moments by documented contract. - # Construction-time reports are deferred until the new - # destinations are installed: reporting routes through emit, so - # firing mid-build would recurse into configure. - previous = self.destinations - deferred: list[tuple[str, BaseException, dict[str, Any]]] = [] - - def deferred_report( - operation: str, exc: BaseException, **fields: Any - ) -> None: - deferred.append((operation, exc, fields)) - - destinations: list[LogDestination] = [] - for destination_name in self.config.log_destinations or ("stdout",): - try: - destinations.append( - self._build_destination( - destination_name, - build_on_failure=deferred_report, - ) - ) - except BaseException as exc: - deferred_report( - "logging.destination_config", - exc, - destination=destination_name, - ) - if not destinations: - destinations.append(self._stdout_destination()) - deferred_report( - "logging.destination_config", - RuntimeError( - "No configured observability log destination " - "initialized; falling back to stdout." - ), - destination="stdout_fallback", - ) - self.destinations = destinations - # The failure ledger is keyed by id(); clear it with the swap so - # stale entries cannot attach to a new destination via id reuse. - self._consecutive_failures.clear() - self.configured = True - # Close the replaced destinations only after the new ones are - # installed, so their close-phase failure reports (which route - # through emit) still have a sink. - self._close_destinations(previous) - for operation, exc, fields in deferred: - self.on_failure(operation, exc, **fields) - for warning in getattr(self.config, "config_warnings", ()): - self.on_failure("logging.profile_config", ValueError(warning)) - - def emit( - self, - payload: dict[str, Any], - *, - log_type: str, - severity: str, - ) -> None: - emitted_payload = {**payload, "severity": severity} - tripped: list[LogDestination] = [] - for destination in self._ensure_destinations(): - try: - destination.emit( - emitted_payload, - log_type=log_type, - severity=severity, - ) - self._consecutive_failures.pop(id(destination), None) - except BaseException as exc: - failures = ( - self._consecutive_failures.get(id(destination), 0) + 1 - ) - self._consecutive_failures[id(destination)] = failures - if failures >= DESTINATION_FAILURE_LIMIT: - tripped.append(destination) - self.on_failure( - "logging.destination_emit", - exc, - destination=getattr(destination, "name", None), - log_type=log_type, - consecutive_failures=failures, - ) - for destination in tripped: - self._disable_destination(destination) - - def close(self, deadline_seconds: float | None = None) -> None: - """Close every destination that supports closing, best-effort.""" - self._close_destinations(self.destinations, deadline_seconds) - - def _close_destinations( - self, - destinations: list[LogDestination], - deadline_seconds: float | None = None, - ) -> None: - # One deadline covers the whole batch: each close gets whatever - # budget the earlier ones left, so N stuck destinations cannot - # take N times the budget. None means each destination applies - # its own default (a reconfigure, not a bounded shutdown). - deadline = ( - None - if deadline_seconds is None - else time.monotonic() + max(0.0, deadline_seconds) - ) - for destination in destinations: - remaining = ( - None - if deadline is None - else max(0.0, deadline - time.monotonic()) - ) - close_destination( - destination, - on_failure=self.on_failure, - deadline_seconds=remaining, - ) - - def _ensure_destinations(self) -> list[LogDestination]: - if not self.configured: - try: - self.configure() - except BaseException as exc: - self.destinations = [self._stdout_destination()] - self.configured = True - self.on_failure("logging.destination_config", exc) - return self.destinations - - def _disable_destination(self, destination: LogDestination) -> None: - self.destinations = [ - existing - for existing in self.destinations - if existing is not destination - ] - self._consecutive_failures.pop(id(destination), None) - self.on_failure( - "logging.destination_disabled", - RuntimeError( - "Disabling observability log destination after " - f"{DESTINATION_FAILURE_LIMIT} consecutive emit failures." - ), - destination=getattr(destination, "name", None), - ) - if not self.destinations: - self.destinations.append(self._stdout_destination()) - - def _build_destination( - self, - destination_name: str, - *, - build_on_failure: Callable[..., None], - ) -> LogDestination: - strategy = destination_strategy(destination_name) - if strategy is None: - raise ValueError( - f"Unknown observability log destination: {destination_name}" - ) - destination = strategy.factory( - config=self.config, - loggers=self.loggers, - serializer=self.serializer, - on_failure=build_on_failure, - ) - if strategy.transport == "remote": - # No remote strategy may ever write on a request thread. - destination = QueuedLogDestination( - inner=destination, - on_failure=self.on_failure, - maxsize=self.config.log_queue_maxsize, - close_timeout_seconds=( - self.config.log_queue_close_timeout_seconds - ), - ) - return destination - - def _stdout_destination(self) -> StdoutJsonDestination: - # The fail-open fallback: built directly (registry-free) and - # with reporting suppressed, because this can run from inside a - # failure-reporting path where another report would recurse. - return build_stdout_destination( - config=self.config, - loggers=self.loggers, - serializer=self.serializer, - ) diff --git a/policyengine_observability/destinations/queued.py b/policyengine_observability/destinations/queued.py deleted file mode 100644 index 1b5f30f..0000000 --- a/policyengine_observability/destinations/queued.py +++ /dev/null @@ -1,300 +0,0 @@ -"""Bounded, best-effort background delivery for remote log destinations. - -``QueuedLogDestination`` wraps any ``LogDestination`` so the caller's -thread only ever enqueues: a stdlib ``logging.handlers.QueueListener`` -thread performs the actual writes, so a degraded sink can never stall a -request. Delivery is best-effort by contract — the stdout sibling -destination is the durable record — so a full queue drops the newest -record and counts it, and there is deliberately no circuit breaker, -retry queue, or recovery machinery in this component. - -Mutable state census (any addition needs design review): - -1. ``_queue`` — thread-safe by construction (``queue.Queue``). -2. ``_listener``— started once in ``__init__``, stopped once in ``close``. -3. ``_drops`` — best-effort drop counter with its throttle state. -4. ``_closed`` — one-way flag flipped by ``close``. - -(The handler's write-failure counter is confined to the listener -thread, so it is not shared mutable state.) - -Accepted races, all bounded and within the best-effort contract: - -- An ``emit`` that passes the ``_closed`` check while ``close`` runs can - lose that one record uncounted. -- When ``close`` times out, the daemon listener thread is abandoned; it - keeps draining in the background until process exit. -- A process that forks after construction (e.g. gunicorn ``--preload``) - inherits a dead listener; records drop with ``reason="full"`` until - the child calls ``restart_observability()`` from a post-fork hook. - There are deliberately no fork hooks or pid checks here. -""" - -from __future__ import annotations - -import atexit -import queue as queue_module -import time -from collections.abc import Callable -from dataclasses import dataclass -from datetime import UTC, datetime -from logging.handlers import QueueListener -from typing import Any - -from ..config import ( - DEFAULT_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS, - DEFAULT_LOG_QUEUE_MAXSIZE, -) -from .base import ( - LogDestination, - accepts_keyword, - clamped, - close_destination, - normalize_payload, - safe_report, -) - -MIN_QUEUE_MAXSIZE = 10 -MAX_QUEUE_MAXSIZE = 100_000 -MIN_CLOSE_TIMEOUT_SECONDS = 0.0 -MAX_CLOSE_TIMEOUT_SECONDS = 30.0 -DROP_REPORT_INTERVAL = 100 -WRITE_FAILURE_REPORT_INTERVAL = 100 - - -@dataclass(frozen=True, slots=True) -class _QueuedRecord: - payload: Any - log_type: str - severity: str - enqueued_at: datetime - - -class _ThrottledCounter: - """Count occurrences; say when one should be reported. - - The shared throttle policy for failure-path reporting: the first - occurrence always reports, then every ``interval``-th, so a - persistent problem stays visible without flooding the internal-error - channel. - """ - - __slots__ = ("count", "interval") - - def __init__(self, interval: int) -> None: - self.interval = max(1, int(interval)) - self.count = 0 - - def tick(self) -> int | None: - """Increment; return the count when this occurrence reports.""" - self.count += 1 - if self.count == 1 or self.count % self.interval == 0: - return self.count - return None - - -class _QueuedRecordHandler: - """Duck-typed QueueListener handler: only ``handle`` is ever called. - - The failure counter is confined to the listener thread. A write - failure must never kill the listener, so everything below the emit - is guarded; ``BaseException`` is deliberately not caught (swallowing - ``SystemExit`` on a worker thread is worse than losing the queue — - the consequences of a dead listener are bounded to counted drops). - """ - - def __init__( - self, - inner: LogDestination, - on_failure: Callable[..., None], - *, - forward_timestamp: bool, - report_interval: int = WRITE_FAILURE_REPORT_INTERVAL, - ) -> None: - self.inner = inner - self.on_failure = on_failure - self.forward_timestamp = forward_timestamp - self.failures = _ThrottledCounter(report_interval) - - def handle(self, record: _QueuedRecord) -> None: - try: - if self.forward_timestamp: - self.inner.emit( - record.payload, - log_type=record.log_type, - severity=record.severity, - timestamp=record.enqueued_at, - ) - else: - self.inner.emit( - record.payload, - log_type=record.log_type, - severity=record.severity, - ) - except Exception as exc: - count = self.failures.tick() - if count is None: - return - safe_report( - self.on_failure, - "logging.queue_write", - exc, - destination=getattr(self.inner, "name", None), - log_type=record.log_type, - write_failures_total=count, - ) - - -class _BoundedQueueListener(QueueListener): - """QueueListener whose stop can be given a hard deadline. - - The stdlib ``stop()`` enqueues its sentinel with ``put_nowait`` - (which raises on a jammed bounded queue) and then joins the worker - thread without a timeout. Here one monotonic deadline covers both - the blocking sentinel put and the join; on expiry the daemon thread - is abandoned and ``False`` is returned. - """ - - def stop(self, timeout: float | None = None) -> bool: - thread = self._thread - if thread is None: - return True - if timeout is None: - super().stop() - return True - deadline = time.monotonic() + max(0.0, timeout) - try: - self.queue.put( - self._sentinel, - timeout=max(0.0, deadline - time.monotonic()), - ) - except queue_module.Full: - pass - thread.join(max(0.0, deadline - time.monotonic())) - stopped = not thread.is_alive() - self._thread = None - return stopped - - -class QueuedLogDestination: - def __init__( - self, - *, - inner: LogDestination, - on_failure: Callable[..., None], - maxsize: float = DEFAULT_LOG_QUEUE_MAXSIZE, - close_timeout_seconds: float = DEFAULT_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS, - drop_report_interval: int = DROP_REPORT_INTERVAL, - ) -> None: - self.inner = inner - self.on_failure = on_failure - self.name = f"queued_{getattr(inner, 'name', 'destination')}" - self.maxsize = int( - clamped( - maxsize, - low=MIN_QUEUE_MAXSIZE, - high=MAX_QUEUE_MAXSIZE, - default=DEFAULT_LOG_QUEUE_MAXSIZE, - ) - ) - self.close_timeout_seconds = clamped( - close_timeout_seconds, - low=MIN_CLOSE_TIMEOUT_SECONDS, - high=MAX_CLOSE_TIMEOUT_SECONDS, - default=DEFAULT_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS, - ) - self._queue: queue_module.Queue[_QueuedRecord | None] = ( - queue_module.Queue(self.maxsize) - ) - self._listener = _BoundedQueueListener( - self._queue, - _QueuedRecordHandler( - inner, - on_failure, - forward_timestamp=accepts_keyword(inner.emit, "timestamp"), - ), - ) - self._drops = _ThrottledCounter(drop_report_interval) - self._closed = False - # Construction happens at configure time on the startup thread, - # never lazily on a request thread. - self._listener.start() - atexit.register(self.close) - - def emit( - self, - payload: dict[str, Any], - *, - log_type: str, - severity: str, - ) -> None: - try: - if self._closed: - self._record_drop("closed", log_type) - return - record = _QueuedRecord( - # Snapshot now: callers keep mutating nested structures - # after emit returns, and the write happens later on the - # listener thread. The enqueue time becomes the entry - # timestamp so delayed writes keep event time. - payload=normalize_payload(payload), - log_type=log_type, - severity=severity, - enqueued_at=datetime.now(UTC), - ) - try: - self._queue.put_nowait(record) - except queue_module.Full: - self._record_drop("full", log_type) - except Exception as exc: - self._record_drop("exception", log_type, exc=exc) - - def close(self, deadline_seconds: float | None = None) -> None: - if self._closed: - return - self._closed = True - atexit.unregister(self.close) - deadline = clamped( - deadline_seconds, - low=MIN_CLOSE_TIMEOUT_SECONDS, - high=MAX_CLOSE_TIMEOUT_SECONDS, - default=self.close_timeout_seconds, - ) - drained = self._listener.stop(timeout=deadline) - if not drained: - safe_report( - self.on_failure, - "logging.queue_close_timeout", - TimeoutError( - "Observability log queue did not drain before " - "the close deadline; remaining records are lost." - ), - destination=self.name, - deadline_seconds=deadline, - pending_records=self._queue.qsize(), - ) - # The abandoned listener may still be mid-write; leave the - # inner destination alone rather than closing it underneath - # an active write. - return - close_destination(self.inner, on_failure=self.on_failure) - - def _record_drop( - self, - reason: str, - log_type: str, - exc: BaseException | None = None, - ) -> None: - count = self._drops.tick() - if count is None: - return - safe_report( - self.on_failure, - "logging.queue_drop", - exc or RuntimeError("Observability log queue dropped a record."), - destination=self.name, - log_type=log_type, - reason=reason, - dropped_total=count, - queue_maxsize=self.maxsize, - ) diff --git a/policyengine_observability/destinations/registry.py b/policyengine_observability/destinations/registry.py deleted file mode 100644 index 25b9521..0000000 --- a/policyengine_observability/destinations/registry.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Named destination strategies. - -Backend modules register themselves here so the manager can build -destinations without knowing any backend: an ``inline`` strategy writes -synchronously on the caller's thread (stdout — the durable, dependency- -free record), while every ``remote`` strategy is wrapped in the bounded -queue transport, so no remote write can ever run on a request thread. -""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from typing import Literal - -from .base import LogDestination, normalize_name - -# Factories are called with keyword arguments (config, loggers, -# serializer, on_failure) and may ignore what they do not need. The -# on_failure callable is for construction-time reporting only; reports -# made through it may be deferred until the build completes. -DestinationFactory = Callable[..., LogDestination] - - -@dataclass(frozen=True) -class DestinationStrategy: - factory: DestinationFactory - transport: Literal["inline", "remote"] - # Config attribute names that must resolve truthy for the strategy - # to be usable. Profiles naming this strategy downgrade gracefully - # (with a warning) when a requirement is missing, instead of failing - # at construction and falling back with a config-failure report. - required_config: tuple[str, ...] = () - - -_STRATEGIES: dict[str, DestinationStrategy] = {} - - -def register_destination( - name: str, - factory: DestinationFactory, - *, - transport: Literal["inline", "remote"], - aliases: tuple[str, ...] = (), - required_config: tuple[str, ...] = (), -) -> None: - strategy = DestinationStrategy( - factory=factory, - transport=transport, - required_config=required_config, - ) - for key in (name, *aliases): - _STRATEGIES[normalize_name(key)] = strategy - - -def destination_strategy(name: str) -> DestinationStrategy | None: - return _STRATEGIES.get(normalize_name(name)) diff --git a/policyengine_observability/destinations/stdout.py b/policyengine_observability/destinations/stdout.py deleted file mode 100644 index 2f562f5..0000000 --- a/policyengine_observability/destinations/stdout.py +++ /dev/null @@ -1,138 +0,0 @@ -from __future__ import annotations - -import logging -from collections.abc import Callable, Mapping -from typing import Any - -from .base import normalize_name, normalize_payload, safe_report -from .registry import register_destination - -# A stdout formatter shapes the normalized payload into the JSON line a -# platform's log agent expects. Formatters are registered by name so -# backend modules can contribute agent-native shapes without the core -# knowing about any backend; "plain" is the built-in default. Factories -# receive the ObservabilityConfig so a formatter can close over settings -# it needs (duck-typed to keep this module config-agnostic). -StdoutFormatter = Callable[..., dict[str, Any]] -StdoutFormatterFactory = Callable[[Any], StdoutFormatter] - -_FORMATTER_FACTORIES: dict[str, StdoutFormatterFactory] = {} - - -def register_stdout_formatter( - name: str, factory: StdoutFormatterFactory -) -> None: - _FORMATTER_FACTORIES[normalize_name(name)] = factory - - -def resolve_stdout_formatter( - config: Any, - on_failure: Callable[..., None] | None = None, -) -> StdoutFormatter: - raw = getattr(config, "stdout_format", None) or "plain" - factory = _FORMATTER_FACTORIES.get(normalize_name(raw)) - if factory is None: - # Falling back must not lose the record, but a typo'd format - # name should not pass silently either — the agent-native shape - # it named would just quietly never appear. - if on_failure is not None: - safe_report( - on_failure, - "logging.stdout_format", - ValueError(f"Unknown stdout format {raw!r}; using plain."), - ) - factory = _FORMATTER_FACTORIES["plain"] - try: - return factory(config) - except Exception as exc: - # Stdout is the fail-open record, and this resolver also runs on - # the manager's last-resort fallback path: a registered factory - # that raises must degrade to the built-in plain formatter (not - # the registry entry, which could be the broken one), never - # break configure or startup. - if on_failure is not None: - safe_report( - on_failure, - "logging.stdout_format", - exc, - stdout_format=raw, - ) - return _plain_formatter_factory(config) - - -def _plain_formatter_factory(config: Any) -> StdoutFormatter: - def format_plain( - payload: dict[str, Any], *, log_type: str, severity: str - ) -> dict[str, Any]: - return payload - - return format_plain - - -register_stdout_formatter("plain", _plain_formatter_factory) - - -class StdoutJsonDestination: - name = "stdout" - - def __init__( - self, - *, - loggers: Mapping[str, logging.Logger], - serializer: Callable[[dict[str, Any]], str], - formatter: StdoutFormatter | None = None, - ) -> None: - self.loggers = loggers - self.serializer = serializer - self.formatter = formatter or _plain_formatter_factory(None) - - def emit( - self, - payload: dict[str, Any], - *, - log_type: str, - severity: str, - ) -> None: - normalized = normalize_payload(payload) - try: - # Formatters receive (and may mutate) the private normalized - # copy. Stdout is the fallback sink, so a broken formatter - # degrades to the unformatted line rather than losing the - # record or tripping the breaker. - formatted = self.formatter( - normalized, log_type=log_type, severity=severity - ) - except Exception: - formatted = normalized - message = self.serializer(formatted) - logger = self.loggers.get(log_type) or self.loggers["event"] - if severity in {"ERROR", "CRITICAL"}: - logger.error(message) - elif severity == "WARNING": - logger.warning(message) - else: - logger.info(message) - - -def build_stdout_destination( - *, - config: Any, - loggers: Any, - serializer: Any, - on_failure: Callable[..., None] | None = None, - **_: Any, -) -> StdoutJsonDestination: - """The one place a configured stdout destination is assembled. - - Used both as the registered ``stdout`` strategy factory and by the - manager's fail-open fallback, so formatter resolution can never - diverge between the two paths. - """ - return StdoutJsonDestination( - loggers=loggers, - serializer=serializer, - formatter=resolve_stdout_formatter(config, on_failure=on_failure), - ) - - -register_destination("stdout", build_stdout_destination, transport="inline") diff --git a/policyengine_observability/diagnostics.py b/policyengine_observability/diagnostics.py new file mode 100644 index 0000000..eb6e84f --- /dev/null +++ b/policyengine_observability/diagnostics.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import json +import sys +import threading +from collections import Counter +from collections.abc import Callable +from datetime import UTC, datetime +from time import monotonic +from typing import Any + + +class Diagnostics: + """Rate-limited local reporting that never enters remote delivery.""" + + def __init__( + self, + *, + interval_seconds: float = 60.0, + stderr: Any = None, + ) -> None: + self._interval_seconds = max(1.0, interval_seconds) + self._stderr = stderr or sys.stderr + self._last_report: dict[str, float] = {} + self._counts: Counter[str] = Counter() + self._listeners: list[Callable[[str, int], None]] = [] + self._lock = threading.Lock() + + def increment(self, name: str, value: int = 1) -> None: + with self._lock: + self._counts[name] += value + listeners = tuple(self._listeners) + for listener in listeners: + try: + listener(name, value) + except Exception: + continue + + def add_listener(self, listener: Callable[[str, int], None]) -> None: + with self._lock: + self._listeners.append(listener) + + def count(self, name: str) -> int: + with self._lock: + return self._counts[name] + + def snapshot(self) -> dict[str, int]: + with self._lock: + return dict(self._counts) + + def report( + self, + operation: str, + error: Exception | str, + **fields: Any, + ) -> None: + self.increment(f"failure.{operation}") + now = monotonic() + with self._lock: + previous = self._last_report.get(operation) + if ( + previous is not None + and now - previous < self._interval_seconds + ): + return + self._last_report[operation] = now + + record = { + "schema_version": "policyengine.observability.internal.v1", + "timestamp": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + "severity": "ERROR", + "event.name": "observability.internal_failure", + "operation": operation, + "error.type": type(error).__name__, + "error.message": _safe_text(error), + **{str(key): _safe_scalar(value) for key, value in fields.items()}, + } + try: + print( + json.dumps(record, sort_keys=True, ensure_ascii=True), + file=self._stderr, + flush=True, + ) + except Exception: + return + + +def _safe_text(value: Any) -> str: + try: + return str(value)[:2_048] + except Exception: + return "" + + +def _safe_scalar(value: Any) -> str | int | float | bool | None: + if value is None or isinstance(value, (str, int, float, bool)): + return value + return _safe_text(value) diff --git a/policyengine_observability/destinations/google_credentials.py b/policyengine_observability/google_credentials.py similarity index 75% rename from policyengine_observability/destinations/google_credentials.py rename to policyengine_observability/google_credentials.py index bed34be..50efb43 100644 --- a/policyengine_observability/destinations/google_credentials.py +++ b/policyengine_observability/google_credentials.py @@ -18,32 +18,6 @@ GOOGLE_CREDENTIAL_SCOPES = ("https://www.googleapis.com/auth/cloud-platform",) -def configure_google_application_credentials( - *, - credentials_json_env: str = "GCP_CREDENTIALS_JSON", - application_credentials_env: str = "GOOGLE_APPLICATION_CREDENTIALS", - credentials_path: Path | None = None, -) -> Path | None: - try: - existing_path = os.getenv(application_credentials_env) - if existing_path: - return Path(existing_path) - - path = _materialize_json_credentials( - credentials_json_env=credentials_json_env, - credentials_path=credentials_path, - ) - if path is None: - path = _materialize_workload_identity_credentials() - if path is None: - return None - - os.environ[application_credentials_env] = str(path) - return path - except Exception: - return None - - def load_google_credentials( *, credentials_json_env: str = "GCP_CREDENTIALS_JSON", @@ -51,14 +25,16 @@ def load_google_credentials( credentials_path: Path | None = None, prefer_workload_identity: bool = False, ) -> Any | None: + """Load Google credentials without performing a network request.""" + try: path: Path | None = None if prefer_workload_identity: path = _materialize_workload_identity_credentials() if path is None: - existing_path = os.getenv(application_credentials_env) - if existing_path: - path = Path(existing_path) + configured_path = os.getenv(application_credentials_env) + if configured_path: + path = Path(configured_path) if path is None: path = _materialize_json_credentials( credentials_json_env=credentials_json_env, @@ -68,7 +44,6 @@ def load_google_credentials( path = _materialize_workload_identity_credentials() if path is None: return None - return _load_credentials_from_file(path) except Exception: return None @@ -82,9 +57,7 @@ def _materialize_json_credentials( credentials_json = os.getenv(credentials_json_env) if not credentials_json: return None - json.loads(credentials_json) - path = credentials_path or Path(tempfile.gettempdir()).joinpath( "policyengine-observability-gcp.json" ) @@ -98,20 +71,23 @@ def _materialize_workload_identity_credentials() -> Path | None: provider = os.getenv(WORKLOAD_IDENTITY_PROVIDER_ENV) if not token or not provider: return None - directory = Path(tempfile.gettempdir()) token_path = directory / "policyengine-observability-oidc.jwt" config_path = directory / "policyengine-observability-wif.json" - token_path.write_text(token) token_path.chmod(0o600) - config = _external_account_config( - provider=provider, - token_path=token_path, - service_account_email=os.getenv(SERVICE_ACCOUNT_EMAIL_ENV), - token_url=os.getenv(STS_TOKEN_URL_ENV) or DEFAULT_STS_TOKEN_URL, + config_path.write_text( + json.dumps( + _external_account_config( + provider=provider, + token_path=token_path, + service_account_email=os.getenv(SERVICE_ACCOUNT_EMAIL_ENV), + token_url=( + os.getenv(STS_TOKEN_URL_ENV) or DEFAULT_STS_TOKEN_URL + ), + ) + ) ) - config_path.write_text(json.dumps(config)) config_path.chmod(0o600) return config_path @@ -119,25 +95,19 @@ def _materialize_workload_identity_credentials() -> Path | None: def _load_credentials_from_file(path: Path) -> Any: config = json.loads(path.read_text()) scopes = list(GOOGLE_CREDENTIAL_SCOPES) - if config.get("type") == "external_account": from google.auth import identity_pool return identity_pool.Credentials.from_info(config, scopes=scopes) - if config.get("type") == "service_account": from google.oauth2.service_account import Credentials - return Credentials.from_service_account_file( - str(path), - scopes=scopes, - ) + return Credentials.from_service_account_file(str(path), scopes=scopes) import google.auth credentials, _project = google.auth.load_credentials_from_file( - str(path), - scopes=scopes, + str(path), scopes=scopes ) return credentials diff --git a/policyengine_observability/integrations/__init__.py b/policyengine_observability/integrations/__init__.py index cbddb89..e2ec12a 100644 --- a/policyengine_observability/integrations/__init__.py +++ b/policyengine_observability/integrations/__init__.py @@ -1 +1,5 @@ """Optional instrumentation integrations.""" + +from .httpx import instrument_httpx + +__all__ = ["instrument_httpx"] diff --git a/policyengine_observability/integrations/httpx.py b/policyengine_observability/integrations/httpx.py index 696d1cd..6a71663 100644 --- a/policyengine_observability/integrations/httpx.py +++ b/policyengine_observability/integrations/httpx.py @@ -1,8 +1,55 @@ from __future__ import annotations -from ..runtime import ObservabilityRuntime, observability_runtime +from typing import Any +from ..runtime import ObservabilityRuntime -def instrument_httpx(runtime: ObservabilityRuntime | None = None) -> None: - runtime = runtime or observability_runtime() - runtime.instrument_httpx() +_RUNTIME_ATTRIBUTE = "_policyengine_observability_runtime" + + +def instrument_httpx(client: Any, runtime: ObservabilityRuntime) -> Any: + """Add correlation headers to requests from one supplied HTTPX client.""" + + existing = getattr(client, _RUNTIME_ATTRIBUTE, None) + if existing is runtime: + return client + if existing is not None: + runtime.diagnostics.report( + "httpx.already_instrumented", + "The HTTPX client is already associated with another runtime.", + ) + return client + + try: + import httpx + + if isinstance(client, httpx.AsyncClient): + + async def inject_async(request: httpx.Request) -> None: + _inject(request, runtime) + + hook = inject_async + elif isinstance(client, httpx.Client): + + def inject_sync(request: httpx.Request) -> None: + _inject(request, runtime) + + hook = inject_sync + else: + raise TypeError("client must be httpx.Client or httpx.AsyncClient") + + client.event_hooks.setdefault("request", []).append(hook) + setattr(client, _RUNTIME_ATTRIBUTE, runtime) + except Exception as exc: + runtime.diagnostics.report("httpx.instrument", exc) + return client + + +def _inject(request: Any, runtime: ObservabilityRuntime) -> None: + try: + headers: dict[str, str] = {} + runtime.inject_http_headers(headers) + for key, value in headers.items(): + request.headers[key] = value + except Exception as exc: + runtime.diagnostics.report("httpx.request_inject", exc) diff --git a/policyengine_observability/logging.py b/policyengine_observability/logging.py deleted file mode 100644 index 666d4ba..0000000 --- a/policyengine_observability/logging.py +++ /dev/null @@ -1,380 +0,0 @@ -"""Structured log emission and observability-failure handling.""" - -from __future__ import annotations - -import json -import logging -import sys -import traceback -from datetime import UTC, datetime -from typing import TYPE_CHECKING, Any - -from .context import ( - ErrorRecord, - OperationObservabilityContext, - RequestObservabilityContext, - _metric_attrs, -) - -if TYPE_CHECKING: - from .runtime import ObservabilityRuntime - - -class PlainMessageFormatter(logging.Formatter): - def format(self, record: logging.LogRecord) -> str: - return record.getMessage() - - -def configure_plain_logger(logger: logging.Logger, level: int) -> None: - logger.setLevel(level) - logger.propagate = False - if not logger.handlers: - handler = logging.StreamHandler() - handler.setFormatter(PlainMessageFormatter()) - logger.addHandler(handler) - - -REQUEST_LOGGER_NAME = "policyengine_observability.requests" -OPERATION_LOGGER_NAME = "policyengine_observability.operations" -EVENT_LOGGER_NAME = "policyengine_observability.events" -INTERNAL_LOGGER_NAME = "policyengine_observability.internal" - -REQUEST_LOGGER = logging.getLogger(REQUEST_LOGGER_NAME) -OPERATION_LOGGER = logging.getLogger(OPERATION_LOGGER_NAME) -EVENT_LOGGER = logging.getLogger(EVENT_LOGGER_NAME) -INTERNAL_LOGGER = logging.getLogger(INTERNAL_LOGGER_NAME) - - -class LogEmitter: - def __init__(self, runtime: ObservabilityRuntime) -> None: - self.runtime = runtime - - def record_error( - self, - exc: BaseException, - *, - handled: bool, - status_code: int | None = None, - include_stack: bool = True, - ) -> None: - if not self.runtime.enabled: - return - try: - context = self.runtime.current_context() - operation = self.runtime.current_operation() - error_record = ErrorRecord( - type=type(exc).__name__, - message=self.runtime._safe_str(exc), - handled=handled, - stack=( - self.runtime._safe_traceback(exc) - if include_stack - else None - ), - ) - if context is not None: - if status_code is not None: - context.status_code = status_code - context.error = error_record - self.runtime.record_error_metric( - context.metric_attributes(error_type=type(exc).__name__) - ) - elif operation is not None: - operation.error = error_record - self.runtime.record_error_metric( - operation.metric_attributes(error_type=type(exc).__name__) - ) - else: - return - span = self.runtime._current_span() - if span is not None: - self.runtime._record_exception_on_span( - span, - exc, - handled=handled, - status_code=status_code, - ) - except BaseException as observability_exc: - self.runtime.log_observability_failure( - "request.record_error", - observability_exc, - original_error_type=type(exc).__name__, - ) - - def record_event(self, event: str, **fields: Any) -> None: - if not self.runtime.enabled: - return - try: - context = self.runtime.current_context() - operation = self.runtime.current_operation() - base: dict[str, Any] = { - "schema_version": "policyengine.observability.event.v1", - "event": event, - "service_name": self.runtime.config.service_name, - "service_role": self.runtime.config.service_role, - "environment": self.runtime.config.environment, - "created_at": datetime.now(UTC).isoformat(), - } - if context is not None: - trace_id, span_id = self.runtime._trace_ids() - base.update( - { - "service_name": context.config.service_name, - "service_role": context.config.service_role, - "environment": context.config.environment, - "request_id": context.request_id, - "trace_id": trace_id, - "span_id": span_id, - "route": context.route, - "path": context.path, - } - ) - elif operation is not None: - trace_id, span_id = self.runtime._trace_ids() - base.update( - { - "service_name": operation.config.service_name, - "service_role": operation.config.service_role, - "environment": operation.config.environment, - "operation": operation.name, - "flavor": operation.flavor, - "trace_id": trace_id, - "span_id": span_id, - } - ) - clean_fields = { - key: value - for key, value in fields.items() - if value is not None - } - base.update(clean_fields) - self.runtime._emit_structured_log( - base, - log_type="event", - severity="INFO", - ) - self.runtime._add_span_event(event, clean_fields) - if event.startswith("modal_") or "fallback" in event: - attrs = ( - context.metric_attributes(event=event) - if context - else operation.metric_attributes(event=event) - if operation - else _metric_attrs( - {"event": event}, - self.runtime.config.metric_attribute_keys, - ) - ) - self.runtime.record_failover_event_metric(attrs) - except BaseException as exc: - self.runtime.log_observability_failure( - "request.record_event", - exc, - event_name=event, - ) - - def emit_request_log(self, context: RequestObservabilityContext) -> None: - if not self.runtime.enabled: - return - try: - if context.emitted: - return - context.emitted = True - if ( - context.internal_dispatch - or not context.config.request_logs_enabled - ): - return - trace_id, span_id = self.runtime._trace_ids() - payload = context.as_log_record( - trace_id=trace_id, - span_id=span_id, - ) - self.runtime._emit_structured_log( - payload, - log_type="request", - severity=self.runtime._severity_for_log_record(payload), - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "request.emit_request_log", - exc, - request_id=getattr(context, "request_id", None), - ) - - def emit_operation_log( - self, - operation: OperationObservabilityContext, - ) -> None: - if not self.runtime.enabled: - return - try: - if operation.emitted: - return - operation.emitted = True - trace_id, span_id = self.runtime._trace_ids() - payload = operation.as_log_record( - trace_id=trace_id, - span_id=span_id, - ) - self.runtime._emit_structured_log( - payload, - log_type="operation", - severity=self.runtime._severity_for_log_record(payload), - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "operation.emit_log", - exc, - operation=getattr(operation, "name", None), - ) - - def log_observability_failure( - self, - operation: str, - exc: BaseException, - **fields: Any, - ) -> None: - payload = self.runtime._internal_error_payload( - operation, exc, **fields - ) - if self.runtime._emitting_internal_error: - self.runtime._write_stderr(payload) - return - self.runtime._emitting_internal_error = True - try: - self.runtime._emit_structured_log( - payload, - log_type="internal", - severity="ERROR", - ) - except BaseException: - self.runtime._write_stderr(payload) - finally: - self.runtime._emitting_internal_error = False - - def _configure_loggers(self) -> None: - for logger in ( - REQUEST_LOGGER, - OPERATION_LOGGER, - EVENT_LOGGER, - INTERNAL_LOGGER, - ): - configure_plain_logger(logger, self.runtime.config.log_level) - - def _emit_structured_log( - self, - payload: dict[str, Any], - *, - log_type: str, - severity: str, - ) -> None: - try: - if not self.runtime.log_destination_manager.configured: - self.runtime._configure_loggers() - self.runtime.log_destination_manager.emit( - payload, - log_type=log_type, - severity=severity, - ) - except BaseException as exc: - if self.runtime._emitting_internal_error: - self.runtime._write_stderr(payload) - else: - self.runtime.log_observability_failure( - "logging.emit", - exc, - log_type=log_type, - ) - - def _handle_destination_failure( - self, - operation: str, - exc: BaseException, - **fields: Any, - ) -> None: - if self.runtime._emitting_internal_error: - self.runtime._write_stderr( - self.runtime._internal_error_payload(operation, exc, **fields) - ) - return - self.runtime.log_observability_failure(operation, exc, **fields) - - def _severity_for_log_record(self, payload: dict[str, Any]) -> str: - status_code = self.runtime._int_or_none(payload.get("status_code")) - error = payload.get("error") - handled = error.get("handled") if isinstance(error, dict) else None - if status_code is not None and status_code >= 500: - return "ERROR" - if error is not None and handled is False: - return "ERROR" - if error is not None or ( - status_code is not None and status_code >= 400 - ): - return "WARNING" - return "INFO" - - def _int_or_none(self, value: Any) -> int | None: - try: - return int(value) - except (TypeError, ValueError): - return None - - def _internal_error_payload( - self, - operation: str, - exc: BaseException, - **fields: Any, - ) -> dict[str, Any]: - payload = { - "schema_version": "policyengine.observability.internal_error.v1", - "event": "observability_internal_error", - "service_name": self.runtime.config.service_name, - "service_role": self.runtime.config.service_role, - "environment": self.runtime.config.environment, - "created_at": datetime.now(UTC).isoformat(), - "operation": operation, - "error": { - "type": type(exc).__name__, - "message": self.runtime._safe_str(exc), - "stack": self.runtime._safe_traceback(exc), - }, - } - payload.update( - {key: value for key, value in fields.items() if value is not None} - ) - return payload - - def _safe_str(self, value: Any) -> str: - try: - return str(value) - except BaseException: - return f"" - - def _safe_traceback(self, exc: BaseException) -> str: - try: - return "".join( - traceback.format_exception(type(exc), exc, exc.__traceback__) - ) - except BaseException: - return "" - - def _json(self, payload: dict[str, Any]) -> str: - try: - return json.dumps(payload, sort_keys=True, default=str) - except BaseException: - return json.dumps( - { - "schema_version": "policyengine.observability.internal_error.v1", - "event": "observability_internal_error", - "created_at": datetime.now(UTC).isoformat(), - "operation": "observability.failure_json", - }, - sort_keys=True, - ) - - def _write_stderr(self, payload: dict[str, Any]) -> None: - try: - sys.stderr.write(self.runtime._json(payload) + "\n") - except BaseException: - return diff --git a/policyengine_observability/otel.py b/policyengine_observability/otel.py new file mode 100644 index 0000000..e94f553 --- /dev/null +++ b/policyengine_observability/otel.py @@ -0,0 +1,701 @@ +from __future__ import annotations + +import os +import threading +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any, cast +from urllib.parse import urlparse + +from .config import ObservabilityConfig, OTelConfig +from .diagnostics import Diagnostics +from .schema import metric_attributes + + +@dataclass(slots=True) +class SpanHandle: + manager: Any + span: Any + + +class OTelRuntime: + def __init__( + self, + config: ObservabilityConfig, + diagnostics: Diagnostics, + *, + queue_depth: Callable[[], int] | None = None, + ) -> None: + self.config = config + self.diagnostics = diagnostics + self._queue_depth_callback = queue_depth or (lambda: 0) + self.tracer: Any | None = None + self.meter: Any | None = None + self._tracer_provider: Any | None = None + self._meter_provider: Any | None = None + self._owns_tracer_provider = False + self._owns_meter_provider = False + self._request_count: Any | None = None + self._request_duration: Any | None = None + self._operation_count: Any | None = None + self._operation_duration: Any | None = None + self._error_count: Any | None = None + self._dropped_count: Any | None = None + self._exporter_failure_count: Any | None = None + self._queue_depth: Any | None = None + self._configure() + + def _configure(self) -> None: + if not self.config.otel.enabled or not self.config.identity_complete: + return + try: + from opentelemetry import metrics, trace + + if self.config.otel.provider_mode == "external": + self._tracer_provider = trace.get_tracer_provider() + self._meter_provider = metrics.get_meter_provider() + else: + self._configure_owned_providers() + if self._tracer_provider is not None: + self.tracer = self._tracer_provider.get_tracer( + "policyengine-observability", "2.0.0" + ) + if self._meter_provider is not None: + self.meter = self._meter_provider.get_meter( + "policyengine-observability", "2.0.0" + ) + self._configure_instruments() + except Exception as exc: + self.diagnostics.report("otel.configure", exc) + self.tracer = None + self.meter = None + + def _configure_owned_providers(self) -> None: + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.sampling import ( + ParentBased, + TraceIdRatioBased, + ) + + resource = Resource.create(self.resource_attributes()) + readers: list[Any] = [] + otel = self.config.otel + if otel.endpoint and otel.metrics_enabled: + from opentelemetry.sdk.metrics.export import ( + PeriodicExportingMetricReader, + ) + + readers.append( + PeriodicExportingMetricReader( + _lazy_metric_exporter_class()( + lambda: _build_metric_exporter(otel), + self.diagnostics, + ), + export_interval_millis=max( + 1_000, + otel.metric_export_interval_seconds * 1_000, + ), + export_timeout_millis=max( + 1, + otel.export_timeout_seconds * 1_000, + ), + ) + ) + self._meter_provider = MeterProvider( + metric_readers=readers, + resource=resource, + shutdown_on_exit=False, + ) + self._owns_meter_provider = True + + self._tracer_provider = TracerProvider( + sampler=ParentBased( + TraceIdRatioBased(min(max(otel.sampling_ratio, 0.0), 1.0)) + ), + resource=resource, + shutdown_on_exit=False, + meter_provider=self._meter_provider, + ) + self._owns_tracer_provider = True + if otel.endpoint and otel.traces_enabled: + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + batch_size = min( + max(1, otel.span_batch_size), + max(1, otel.span_queue_capacity), + ) + self._tracer_provider.add_span_processor( + BatchSpanProcessor( + cast( + Any, + _LazySpanExporter( + lambda: _build_span_exporter(otel), + self.diagnostics, + ), + ), + max_queue_size=max(1, otel.span_queue_capacity), + max_export_batch_size=batch_size, + schedule_delay_millis=max( + 1, + otel.span_schedule_delay_seconds * 1_000, + ), + export_timeout_millis=max( + 1, + otel.export_timeout_seconds * 1_000, + ), + meter_provider=self._meter_provider, + ) + ) + + def _configure_instruments(self) -> None: + meter = self.meter + if meter is None: + return + self._request_count = meter.create_counter( + "policyengine.request.count" + ) + self._request_duration = meter.create_histogram( + "policyengine.request.duration", + unit="s", + ) + self._operation_count = meter.create_counter( + "policyengine.operation.count" + ) + self._operation_duration = meter.create_histogram( + "policyengine.operation.duration", + unit="s", + ) + self._error_count = meter.create_counter("policyengine.error.count") + self._dropped_count = meter.create_counter( + "policyengine.telemetry.dropped" + ) + self._exporter_failure_count = meter.create_counter( + "policyengine.telemetry.exporter.failure" + ) + self._queue_depth = meter.create_observable_gauge( + "policyengine.telemetry.queue.depth", + callbacks=[self._observe_queue_depth], + ) + + def _observe_queue_depth(self, _options: Any) -> list[Any]: + try: + from opentelemetry.metrics import Observation + + return [Observation(max(0, int(self._queue_depth_callback())))] + except Exception as exc: + self.diagnostics.report("otel.queue_depth", exc) + return [] + + def resource_attributes(self) -> dict[str, str]: + service = self.config.service + deployment = self.config.deployment + values = { + "service.name": service.name, + "service.namespace": service.namespace, + "service.version": service.version, + "service.instance.id": deployment.instance_id, + "deployment.environment.name": deployment.environment, + "cloud.platform": deployment.platform, + "cloud.region": deployment.region, + "policyengine.service.role": service.role, + } + return {key: value for key, value in values.items() if value} + + def start_span( + self, + name: str, + *, + kind: Any = None, + attributes: Mapping[str, Any] | None = None, + parent_context: Any = None, + links: Sequence[Any] | None = None, + ) -> SpanHandle | None: + if self.tracer is None: + return None + try: + kwargs: dict[str, Any] = { + "attributes": dict(attributes or {}), + "record_exception": False, + "set_status_on_exception": False, + } + if kind is not None: + kwargs["kind"] = kind + if parent_context is not None: + kwargs["context"] = parent_context + if links: + kwargs["links"] = list(links) + manager = self.tracer.start_as_current_span(name, **kwargs) + return SpanHandle(manager=manager, span=manager.__enter__()) + except Exception as exc: + self.diagnostics.report("otel.span_start", exc, span=name) + return None + + def end_span( + self, + handle: SpanHandle | None, + error: BaseException | None = None, + ) -> None: + if handle is None: + return + try: + if error is not None: + from opentelemetry.trace import Status, StatusCode + + if isinstance(error, Exception): + handle.span.record_exception(error) + handle.span.set_status(Status(StatusCode.ERROR)) + handle.manager.__exit__( + type(error) if error is not None else None, + error, + error.__traceback__ if error is not None else None, + ) + except Exception as exc: + self.diagnostics.report("otel.span_end", exc) + + def set_span_attributes(self, values: Mapping[str, Any]) -> None: + try: + from opentelemetry import trace + + span = trace.get_current_span() + if not span.is_recording(): + return + for key, value in values.items(): + span.set_attribute(key, value) + except Exception as exc: + self.diagnostics.report("otel.span_attribute", exc) + + def current_correlation(self) -> dict[str, Any]: + try: + from opentelemetry import trace + + span_context = trace.get_current_span().get_span_context() + if not span_context.is_valid: + return {} + return { + "trace_id": format(span_context.trace_id, "032x"), + "span_id": format(span_context.span_id, "016x"), + "trace_sampled": bool(span_context.trace_flags.sampled), + } + except Exception: + return {} + + def extract(self, carrier: Mapping[str, str]) -> Any: + try: + from opentelemetry.trace.propagation.tracecontext import ( + TraceContextTextMapPropagator, + ) + + return TraceContextTextMapPropagator().extract(carrier=carrier) + except Exception as exc: + self.diagnostics.report("otel.context_extract", exc) + return None + + def inject(self, carrier: dict[str, str]) -> None: + try: + from opentelemetry.trace.propagation.tracecontext import ( + TraceContextTextMapPropagator, + ) + + TraceContextTextMapPropagator().inject(carrier=carrier) + except Exception as exc: + self.diagnostics.report("otel.context_inject", exc) + + def remote_span_context(self, carrier: Mapping[str, str]) -> Any | None: + try: + from opentelemetry import trace + + context = self.extract(carrier) + if context is None: + return None + span_context = trace.get_current_span(context).get_span_context() + return span_context if span_context.is_valid else None + except Exception: + return None + + def link(self, span_context: Any) -> Any | None: + if span_context is None: + return None + try: + from opentelemetry.trace import Link + + return Link(span_context) + except Exception: + return None + + def empty_context(self) -> Any | None: + try: + from opentelemetry.context import Context + + return Context() + except Exception: + return None + + def record_request( + self, + duration_seconds: float, + attributes: Mapping[str, Any], + ) -> None: + bounded = metric_attributes(attributes, self.config) + self._safe_metric(self._request_count, "add", 1, bounded) + self._safe_metric( + self._request_duration, + "record", + duration_seconds, + bounded, + ) + + def record_operation( + self, + duration_seconds: float, + attributes: Mapping[str, Any], + ) -> None: + bounded = metric_attributes(attributes, self.config) + self._safe_metric(self._operation_count, "add", 1, bounded) + self._safe_metric( + self._operation_duration, + "record", + duration_seconds, + bounded, + ) + + def record_error(self, attributes: Mapping[str, Any]) -> None: + self._safe_metric( + self._error_count, + "add", + 1, + metric_attributes(attributes, self.config), + ) + + def record_dropped(self, kind: str, count: int = 1) -> None: + self._safe_metric( + self._dropped_count, + "add", + count, + {"operation.kind": kind}, + ) + + def record_exporter_failure(self, kind: str, count: int = 1) -> None: + self._safe_metric( + self._exporter_failure_count, + "add", + count, + {"operation.kind": kind}, + ) + + def _safe_metric( + self, + instrument: Any | None, + method: str, + value: int | float, + attributes: Mapping[str, Any], + ) -> None: + if instrument is None: + return + try: + getattr(instrument, method)(value, attributes=dict(attributes)) + except Exception as exc: + self.diagnostics.report("otel.metric_record", exc) + + def force_flush(self, timeout_seconds: float) -> None: + timeout_ms = max(0, int(timeout_seconds * 1_000)) + for provider in (self._tracer_provider, self._meter_provider): + force_flush = getattr(provider, "force_flush", None) + if callable(force_flush): + try: + force_flush(timeout_millis=timeout_ms) + except Exception as exc: + self.diagnostics.report("otel.force_flush", exc) + + def shutdown(self, timeout_seconds: float) -> None: + timeout_ms = max(0, int(timeout_seconds * 1_000)) + if self._owns_tracer_provider and self._tracer_provider is not None: + try: + self._tracer_provider.shutdown() + except Exception as exc: + self.diagnostics.report("otel.trace_shutdown", exc) + if self._owns_meter_provider and self._meter_provider is not None: + try: + self._meter_provider.shutdown(timeout_millis=timeout_ms) + except Exception as exc: + self.diagnostics.report("otel.metric_shutdown", exc) + + +class _LazySpanExporter: + def __init__( + self, + factory: Callable[[], Any], + diagnostics: Diagnostics, + ) -> None: + self._factory = factory + self._diagnostics = diagnostics + self._delegate: Any | None = None + self._lock = threading.Lock() + + def export(self, spans: Sequence[Any]) -> Any: + from opentelemetry.sdk.trace.export import SpanExportResult + + try: + return self._get().export(spans) + except Exception as exc: + self._diagnostics.increment("spans.export_failure") + self._diagnostics.report("otel.span_export", exc) + return SpanExportResult.FAILURE + + def force_flush(self, timeout_millis: int = 30_000) -> bool: + try: + delegate = self._delegate + return ( + True + if delegate is None + else bool(delegate.force_flush(timeout_millis)) + ) + except Exception as exc: + self._diagnostics.report("otel.span_flush", exc) + return False + + def shutdown(self) -> None: + try: + if self._delegate is not None: + self._delegate.shutdown() + except Exception as exc: + self._diagnostics.report("otel.span_exporter_shutdown", exc) + + def _get(self) -> Any: + if self._delegate is None: + with self._lock: + if self._delegate is None: + self._delegate = self._factory() + return self._delegate + + +def _lazy_metric_exporter_class(): + from opentelemetry.sdk.metrics.export import MetricExporter + + class LazyMetricExporter(MetricExporter): + def __init__( + self, + factory: Callable[[], Any], + diagnostics: Diagnostics, + ) -> None: + super().__init__() + self._factory = factory + self._diagnostics = diagnostics + self._delegate: Any | None = None + self._lock = threading.Lock() + + def export( + self, + metrics_data: Any, + timeout_millis: float = 10_000, + **kwargs: Any, + ) -> Any: + from opentelemetry.sdk.metrics.export import MetricExportResult + + try: + return self._get().export( + metrics_data, + timeout_millis=timeout_millis, + **kwargs, + ) + except Exception as exc: + self._diagnostics.increment("metrics.export_failure") + self._diagnostics.report("otel.metric_export", exc) + return MetricExportResult.FAILURE + + def force_flush(self, timeout_millis: float = 10_000) -> bool: + try: + delegate = self._delegate + return ( + True + if delegate is None + else bool(delegate.force_flush(timeout_millis)) + ) + except Exception as exc: + self._diagnostics.report("otel.metric_flush", exc) + return False + + def shutdown( + self, timeout_millis: float = 30_000, **kwargs: Any + ) -> None: + try: + if self._delegate is not None: + self._delegate.shutdown( + timeout_millis=timeout_millis, + **kwargs, + ) + except Exception as exc: + self._diagnostics.report("otel.metric_exporter_shutdown", exc) + + def _get(self) -> Any: + if self._delegate is None: + with self._lock: + if self._delegate is None: + self._delegate = self._factory() + return self._delegate + + return LazyMetricExporter + + +def _build_span_exporter(config: OTelConfig) -> Any: + kwargs = _exporter_kwargs(config) + if config.protocol == "http/protobuf": + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, + ) + + kwargs["endpoint"] = _http_signal_endpoint( + config.endpoint or "", "traces" + ) + return OTLPSpanExporter(**kwargs) + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, + ) + + return OTLPSpanExporter(**kwargs) + + +def _build_metric_exporter(config: OTelConfig) -> Any: + kwargs = _exporter_kwargs(config) + if config.protocol == "http/protobuf": + from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, + ) + + kwargs["endpoint"] = _http_signal_endpoint( + config.endpoint or "", "metrics" + ) + return OTLPMetricExporter(**kwargs) + from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, + ) + + return OTLPMetricExporter(**kwargs) + + +def _exporter_kwargs(config: OTelConfig) -> dict[str, Any]: + headers = dict(config.headers) + kwargs: dict[str, Any] = { + "endpoint": config.endpoint, + "headers": headers, + "timeout": max(0.1, min(config.export_timeout_seconds, 60.0)), + } + if config.google_audience: + if config.protocol == "grpc": + kwargs["credentials"] = _google_grpc_credentials( + config.google_audience + ) + else: + kwargs["session"] = _google_http_session( + config.google_audience, + headers, + ) + kwargs.pop("headers", None) + return kwargs + + +def _google_id_token_credentials(audience: str) -> Any: + from google.auth.transport.requests import Request + + modal_token = os.getenv("MODAL_IDENTITY_TOKEN") or os.getenv( + "OBSERVABILITY_GOOGLE_OIDC_TOKEN" + ) + provider = os.getenv("OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER") + service_account = os.getenv("OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL") + if modal_token and provider and service_account: + from google.auth import identity_pool, impersonated_credentials + + source = identity_pool.Credentials.from_info( + { + "type": "external_account", + "audience": _workload_identity_audience(provider), + "subject_token_type": ("urn:ietf:params:oauth:token-type:jwt"), + "token_url": "https://sts.googleapis.com/v1/token", + "credential_source": { + "file": _write_subject_token(modal_token), + "format": {"type": "text"}, + }, + }, + scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + target = impersonated_credentials.Credentials( + source_credentials=source, + target_principal=service_account, + target_scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + return impersonated_credentials.IDTokenCredentials( + target_credentials=target, + target_audience=audience, + include_email=True, + ) + + from google.oauth2.id_token import fetch_id_token_credentials + + return fetch_id_token_credentials(audience, request=Request()) + + +def _google_grpc_credentials(audience: str) -> Any: + import grpc + from google.auth.transport.grpc import AuthMetadataPlugin + from google.auth.transport.requests import Request + + plugin = AuthMetadataPlugin( + _google_id_token_credentials(audience), + Request(), + default_host=urlparse(audience).netloc, + ) + return grpc.composite_channel_credentials( + grpc.ssl_channel_credentials(), + grpc.metadata_call_credentials(plugin), + ) + + +def _google_http_session(audience: str, headers: Mapping[str, str]) -> Any: + from google.auth.transport.requests import AuthorizedSession + + session = AuthorizedSession(_google_id_token_credentials(audience)) + session.headers.update(headers) + return session + + +def _http_signal_endpoint(endpoint: str, signal: str) -> str: + value = endpoint.rstrip("/") + if value.endswith(f"/v1/{signal}"): + return value + return f"{value}/v1/{signal}" + + +def _workload_identity_audience(provider: str) -> str: + value = provider.strip() + if value.startswith("//iam.googleapis.com/"): + return value + if value.startswith("projects/"): + return f"//iam.googleapis.com/{value}" + return value + + +def _write_subject_token(token: str) -> str: + import tempfile + from pathlib import Path + + path = Path(tempfile.gettempdir()) / ( + "policyengine-observability-otel-oidc.jwt" + ) + path.write_text(token) + path.chmod(0o600) + return str(path) + + +def captured_at_is_recent(value: Any, max_age_seconds: float) -> bool: + if not isinstance(value, str): + return False + try: + captured_at = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return False + if captured_at.tzinfo is None: + return False + age = (datetime.now(UTC) - captured_at.astimezone(UTC)).total_seconds() + return 0 <= age <= max_age_seconds diff --git a/policyengine_observability/runtime.py b/policyengine_observability/runtime.py index a2f7248..a37481c 100644 --- a/policyengine_observability/runtime.py +++ b/policyengine_observability/runtime.py @@ -1,540 +1,857 @@ -"""Public runtime interface and component lifecycle.""" - from __future__ import annotations +import inspect +import logging +import re import threading import time -from collections.abc import AsyncIterator, Iterator -from enum import Enum +import uuid +from collections.abc import Mapping +from contextvars import ContextVar, Token +from dataclasses import dataclass, field +from datetime import UTC, datetime +from functools import wraps from typing import Any -from . import _state -from ._metrics import MetricRecorder, _NoOpInstrument -from ._operations import OperationLifecycle -from ._requests import RequestLifecycle -from ._state import ( - OBSERVABILITY_INTERNAL_DISPATCH_HEADER as OBSERVABILITY_INTERNAL_DISPATCH_HEADER, -) -from ._state import ( - REQUEST_ID_HEADER as REQUEST_ID_HEADER, -) -from ._state import ( - TRACEPARENT_HEADER as TRACEPARENT_HEADER, -) -from ._state import ( - ContextState, -) -from ._tracing import TraceRecorder from .config import ObservabilityConfig -from .context import OperationObservabilityContext, RequestObservabilityContext -from .destinations import LogDestinationManager -from .destinations.base import clamped -from .logging import ( - EVENT_LOGGER, - INTERNAL_LOGGER, - OPERATION_LOGGER, - REQUEST_LOGGER, - LogEmitter, -) -from .segments import SegmentRecorder +from .delivery import DeliveryManager +from .diagnostics import Diagnostics +from .otel import OTelRuntime, SpanHandle, captured_at_is_recent +from .schema import build_record, normalize_attributes + +REQUEST_ID_HEADER = "X-PolicyEngine-Request-Id" +TRACEPARENT_HEADER = "traceparent" +TRACESTATE_HEADER = "tracestate" + +_REQUEST_ID_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +@dataclass(slots=True) +class _RequestState: + request_id: str + method: str + route: str + start_time: float + span: SpanHandle | None + token: Token[Any] | None = None + status_code: int | None = None + attributes: dict[str, Any] = field(default_factory=dict) + error: BaseException | None = None + completed: bool = False + + +@dataclass(slots=True) +class _OperationState: + name: str + kind: str + request_id: str | None + start_time: float + span: SpanHandle | None + token: Token[Any] | None = None + attributes: dict[str, Any] = field(default_factory=dict) + error: BaseException | None = None + completed: bool = False + + +@dataclass(slots=True) +class _ChildSpanState: + name: str + start_time: float + span: SpanHandle | None class ObservabilityRuntime: - def __init__( - self, - config: ObservabilityConfig, - *, - segment_registry: type[Enum] | None = None, - ) -> None: + def __init__(self, config: ObservabilityConfig) -> None: self.config = config - self.segment_registry = segment_registry - self.enabled = config.enabled - self.trace = None - self.propagate = None - self.SpanKind = None - self.Status = None - self.StatusCode = None - self.tracer_provider = None - self.meter_provider = None - self.tracer = None - self.meter = None - self.operation_duration = _NoOpInstrument() - self.http_duration = _NoOpInstrument() - self.segment_duration = _NoOpInstrument() - self.calculate_duration = _NoOpInstrument() - self.backend_duration = _NoOpInstrument() - self.operations = _NoOpInstrument() - self.requests = _NoOpInstrument() - self.errors = _NoOpInstrument() - self.rate_limited = _NoOpInstrument() - self.failover_events = _NoOpInstrument() - self.active_requests = _NoOpInstrument() - self._httpx_instrumented = False - self._emitting_internal_error = False - self.log_destination_manager = LogDestinationManager( - config=config, - loggers={ - "request": REQUEST_LOGGER, - "operation": OPERATION_LOGGER, - "event": EVENT_LOGGER, - "internal": INTERNAL_LOGGER, - }, - serializer=self._json, - on_failure=self._handle_destination_failure, + self.diagnostics = Diagnostics() + self._request_state: ContextVar[_RequestState | None] = ContextVar( + f"policyengine_request_{id(self)}", default=None ) - self._context_state = ContextState(self) - self._operations = OperationLifecycle(self) - self._requests = RequestLifecycle(self) - self._segments = SegmentRecorder(self) - self._logging = LogEmitter(self) - self._metrics = MetricRecorder(self) - self._tracing = TraceRecorder(self) - - @classmethod - def disabled(cls) -> ObservabilityRuntime: - return cls(ObservabilityConfig(enabled=False)) - - def configure(self) -> None: - self._configure_loggers() - if not self.enabled: - return - self.log_destination_manager.configure() - if not self.config.otel_enabled: - return - self._configure_otel() - if self.config.instrument_httpx: - self.instrument_httpx() + self._operation_state: ContextVar[_OperationState | None] = ContextVar( + f"policyengine_operation_{id(self)}", default=None + ) + self._shutdown_lock = threading.Lock() + self._closed = False + self._delivery = self._new_delivery() + self._otel = self._new_otel() + self.diagnostics.add_listener(self._record_diagnostic_metric) + self._logging_handlers: list[ + tuple[logging.Logger, ObservabilityLogHandler] + ] = [] + for warning in config.diagnostics(): + self.diagnostics.report("configuration", warning) + if config.logging.capture_standard_library: + instrument_logging( + logging.getLogger(), + self, + replace=config.logging.replace_existing_handlers, + ) - def current_context(self) -> RequestObservabilityContext | None: - return self._context_state.current_context() + def _new_delivery(self) -> DeliveryManager: + try: + return DeliveryManager(self.config, self.diagnostics) + except Exception as exc: + self.diagnostics.report("delivery.configure", exc) + fallback = ObservabilityConfig( + service=self.config.service, + deployment=self.config.deployment, + google_cloud_project_id=self.config.google_cloud_project_id, + logging=self.config.logging.__class__( + stdout_enabled=True, + remote=None, + ), + otel=self.config.otel, + limits=self.config.limits, + application_attribute_keys=( + self.config.application_attribute_keys + ), + dispatch_attribute_keys=self.config.dispatch_attribute_keys, + metric_attribute_keys=self.config.metric_attribute_keys, + sensitive_values=self.config.sensitive_values, + ) + return DeliveryManager(fallback, self.diagnostics) - def current_operation(self) -> OperationObservabilityContext | None: - return self._context_state.current_operation() + def _new_otel(self) -> OTelRuntime: + try: + return OTelRuntime( + self.config, + self.diagnostics, + queue_depth=lambda: self._delivery.queue_depth, + ) + except Exception as exc: + self.diagnostics.report("otel.runtime", exc) + return OTelRuntime( + ObservabilityConfig( + service=self.config.service, + deployment=self.config.deployment, + google_cloud_project_id=( + self.config.google_cloud_project_id + ), + logging=self.config.logging, + otel=self.config.otel.__class__(enabled=False), + limits=self.config.limits, + application_attribute_keys=( + self.config.application_attribute_keys + ), + dispatch_attribute_keys=( + self.config.dispatch_attribute_keys + ), + metric_attribute_keys=self.config.metric_attribute_keys, + sensitive_values=self.config.sensitive_values, + ), + self.diagnostics, + queue_depth=lambda: self._delivery.queue_depth, + ) - def operation(self, name: str, *, flavor: str | None = None, **attrs: Any): - return self._operations.operation(name, flavor=flavor, **attrs) + def _record_diagnostic_metric(self, name: str, value: int) -> None: + if "dropped" in name or name.endswith("queue_full"): + self._otel.record_dropped(name, value) + elif "export_failure" in name: + self._otel.record_exporter_failure(name, value) - def entrypoint( + def operation( self, - name: str | None = None, + name: str, *, - flavor: str | None = None, - **attrs: Any, - ): - return self._operations.entrypoint(name, flavor=flavor, **attrs) + attributes: Mapping[str, Any] | None = None, + remote_context: Mapping[str, Any] | None = None, + independent_retry: bool = False, + aggregate: bool = False, + ) -> _ScopeManager: + return _ScopeManager( + self, + scope_type="operation", + name=name, + attributes=attributes, + remote_context=remote_context, + independent_retry=independent_retry, + aggregate=aggregate, + ) - def start_operation( + def span( self, name: str, *, - flavor: str | None = None, - parent_context: Any = None, - timings: dict[str, float] | None = None, - emit_log: bool = True, - record_metric: bool = True, - **attrs: Any, - ) -> dict[str, Any]: - return self._operations.start_operation( - name, - flavor=flavor, - parent_context=parent_context, - timings=timings, - emit_log=emit_log, - record_metric=record_metric, - **attrs, + attributes: Mapping[str, Any] | None = None, + ) -> _ScopeManager: + return _ScopeManager( + self, + scope_type="span", + name=name, + attributes=attributes, ) - def end_operation( - self, handle: dict[str, Any] | None, error: BaseException | None = None - ) -> None: - return self._operations.end_operation(handle, error) - - def complete_operation( - self, operation: OperationObservabilityContext - ) -> None: - return self._operations.complete_operation(operation) + def _scope_copy(self, manager: _ScopeManager) -> _ScopeManager: + return _ScopeManager( + self, + scope_type=manager.scope_type, + name=manager.name, + attributes=manager.attributes, + remote_context=manager.remote_context, + independent_retry=manager.independent_retry, + aggregate=manager.aggregate, + ) def begin_request( - self, context: RequestObservabilityContext, *, carrier: Any = None - ) -> None: - return self._requests.begin_request(context, carrier=carrier) - - def _begin_request_operation(self, *args: Any, **kwargs: Any) -> Any: - return self._requests._begin_request_operation(*args, **kwargs) - - def finish_request(self, status_code: int) -> dict[str, str]: - return self._requests.finish_request(status_code) - - def prepare_response(self, status_code: int) -> dict[str, str]: - return self._requests.prepare_response(status_code) - - def complete_request(self, status_code: int | None = None) -> None: - return self._requests.complete_request(status_code) - - def update_request_route( - self, *, route: str | None = None, endpoint: str | None = None + self, + *, + headers: Mapping[str, str], + method: str, + route: str, + ) -> str: + request_id = _request_id(_header_value(headers, REQUEST_ID_HEADER)) + try: + parent = self._otel.extract(headers) + span = self._otel.start_span( + f"{method.upper()} {route}", + kind=_span_kind("SERVER"), + parent_context=parent, + attributes={ + "http.request.method": method.upper(), + "http.route": route, + }, + ) + state = _RequestState( + request_id=request_id, + method=method.upper(), + route=route, + start_time=time.perf_counter(), + span=span, + ) + state.token = self._request_state.set(state) + except Exception as exc: + self.diagnostics.report("request.begin", exc) + return request_id + + def update_request_route(self, route: str) -> None: + state = self._request_state.get() + if state is None or state.completed: + return + state.route = route + self._otel.set_span_attributes({"http.route": route}) + + def response_headers(self) -> dict[str, str]: + state = self._request_state.get() + if state is None: + return {} + headers = {REQUEST_ID_HEADER: state.request_id} + carrier: dict[str, str] = {} + self._otel.inject(carrier) + if TRACEPARENT_HEADER in carrier: + headers[TRACEPARENT_HEADER] = carrier[TRACEPARENT_HEADER] + return headers + + def end_request( + self, + *, + status_code: int | None, + error: BaseException | None = None, ) -> None: - return self._requests.update_request_route( - route=route, endpoint=endpoint + state = self._request_state.get() + if state is None or state.completed: + return + state.completed = True + state.status_code = status_code + state.error = error + duration = max(0.0, time.perf_counter() - state.start_time) + outcome = _outcome(status_code, error) + metric_values = self._metric_base() + metric_values.update( + { + "http.route": state.route, + "http.request.method": state.method, + "http.response.status_code_class": _status_class(status_code), + "operation.kind": "request", + "outcome": outcome, + } ) - - def teardown_request(self, exc: BaseException | None = None) -> None: - return self._requests.teardown_request(exc) - - def set_attribute(self, key: str, value: Any) -> None: - return self._requests.set_attribute(key, value) - - def segment(self, name: Any, **attrs: Any) -> Iterator[Any]: - return self._segments.segment(name, **attrs) - - def _segment_context(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._segment_context(*args, **kwargs) - - def asegment(self, name: Any, **attrs: Any) -> AsyncIterator[Any]: - return self._segments.asegment(name, **attrs) - - def collect_timings(self, name: str = "operation", **attrs: Any): - return self._operations.collect_timings(name, **attrs) - - def start_scope( + self._otel.set_span_attributes( + { + "http.route": state.route, + "http.response.status_code": status_code or 0, + "policyengine.request.id": state.request_id, + "policyengine.outcome": outcome, + } + ) + context = { + "request.id": state.request_id, + "http.request.method": state.method, + "http.route": state.route, + "http.response.status_code": status_code, + "duration_ms": round(duration * 1_000, 3), + "outcome": outcome, + **self._otel.current_correlation(), + } + self._emit_record( + severity="ERROR" if outcome == "error" else "INFO", + event_name="request.completed", + context=context, + attributes=state.attributes, + error=error, + ) + self._otel.record_request(duration, metric_values) + if error is not None: + self._otel.record_error(metric_values) + self._otel.end_span(state.span, error) + self._reset_request(state) + + def set_context(self, **attributes: Any) -> None: + safe, omitted = normalize_attributes( + attributes, + self.config, + allowed_keys=( + self.config.application_attribute_keys + | self.config.dispatch_attribute_keys + ), + ) + request = self._request_state.get() + operation = self._operation_state.get() + target = operation or request + if target is not None: + target.attributes.update(safe) + if safe: + self._otel.set_span_attributes(safe) + if omitted: + self.diagnostics.increment("attributes.omitted", omitted) + + def event( self, - timings: dict[str, float], + name: str, *, - name: str = "operation", - parent_context: Any = None, - **attrs: Any, - ) -> dict[str, Any]: - return self._operations.start_scope( - timings, name=name, parent_context=parent_context, **attrs - ) - - def annotate( - self, handle: dict[str, Any] | None = None, **attrs: Any + severity: str = "INFO", + attributes: Mapping[str, Any] | None = None, ) -> None: - return self._operations.annotate(handle, **attrs) + self._emit_record( + severity=severity, + event_name=name, + context=self._active_context_fields(), + attributes=attributes, + ) - def end_scope( - self, handle: dict[str, Any] | None, error: BaseException | None = None + def log( + self, + message: str, + *, + severity: str = "INFO", + attributes: Mapping[str, Any] | None = None, + error: BaseException | None = None, ) -> None: - return self._operations.end_scope(handle, error) - - def mark(self, key: str, ms: float) -> None: - return self._operations.mark(key, ms) - - def mark_ttft(self, key: str = "ttft_ms") -> None: - return self._operations.mark_ttft(key) - - def mark_ttft_attribute(self, key: str = "ttft_ms") -> None: - return self._operations.mark_ttft_attribute(key) + self._emit_record( + severity=severity, + message=message, + context=self._active_context_fields(), + attributes=attributes, + error=error, + ) - def record_error( + def record_exception( self, - exc: BaseException, + error: Exception, *, handled: bool, status_code: int | None = None, - include_stack: bool = True, ) -> None: - return self._logging.record_error( - exc, - handled=handled, - status_code=status_code, - include_stack=include_stack, + request = self._request_state.get() + operation = self._operation_state.get() + if request is not None: + request.error = error + if status_code is not None: + request.status_code = status_code + if operation is not None: + operation.error = error + context = { + **self._active_context_fields(), + "error.handled": handled, + } + if status_code is not None: + context["http.response.status_code"] = status_code + self._emit_record( + severity="ERROR", + event_name="exception.recorded", + context=context, + error=error, + ) + self._otel.record_error( + { + **self._metric_base(), + "operation.kind": "handled" if handled else "unhandled", + "outcome": "error", + } ) - def record_event(self, event: str, **fields: Any) -> None: - return self._logging.record_event(event, **fields) - - def traceparent_header(self) -> str | None: - return self._tracing.traceparent_header() - - def capture_context(self): - return self._tracing.capture_context() - - def emit_request_log(self, context: RequestObservabilityContext) -> None: - return self._logging.emit_request_log(context) + def capture_context(self) -> dict[str, str]: + carrier: dict[str, str] = {} + self._otel.inject(carrier) + captured: dict[str, str] = { + key: value + for key, value in carrier.items() + if key.lower() in {TRACEPARENT_HEADER, TRACESTATE_HEADER} + } + captured["captured_at"] = ( + datetime.now(UTC).isoformat().replace("+00:00", "Z") + ) + request = self._request_state.get() + operation = self._operation_state.get() + if request is not None: + captured["request_id"] = request.request_id + elif operation is not None and operation.request_id: + captured["request_id"] = operation.request_id + source = operation.attributes if operation is not None else {} + if request is not None: + source = {**request.attributes, **source} + for key in self.config.dispatch_attribute_keys: + value = source.get(key) + if isinstance(value, (str, int)): + captured[key] = str(value)[ + : self.config.limits.max_string_length + ] + return captured + + def inject_http_headers(self, headers: dict[str, str]) -> None: + self._otel.inject(headers) + if REQUEST_ID_HEADER not in headers: + request = self._request_state.get() + operation = self._operation_state.get() + request_id = ( + request.request_id + if request is not None + else operation.request_id + if operation is not None + else None + ) + if request_id: + headers[REQUEST_ID_HEADER] = request_id - def emit_operation_log( - self, operation: OperationObservabilityContext - ) -> None: - return self._logging.emit_operation_log(operation) + def shutdown(self) -> None: + with self._shutdown_lock: + if self._closed: + return + self._closed = True + timeout = max(0.0, self.config.limits.shutdown_timeout_seconds) + self._delivery.close(timeout) + _run_bounded( + lambda: self._otel.shutdown(timeout), + timeout, + self.diagnostics, + "otel.shutdown_deadline", + ) + self._remove_logging_handlers() - def record_operation_metric( - self, duration_seconds: float, attributes: dict[str, str] - ) -> None: - return self._metrics.record_operation_metric( - duration_seconds, attributes + def restart_after_snapshot(self) -> None: + with self._shutdown_lock: + timeout = min( + max(0.0, self.config.limits.shutdown_timeout_seconds), 1.0 + ) + try: + self._delivery.close(timeout) + except Exception as exc: + self.diagnostics.report("snapshot.delivery_close", exc) + self._delivery = self._new_delivery() + self._otel = self._new_otel() + self._closed = False + + def _start_operation( + self, + name: str, + attributes: Mapping[str, Any] | None, + remote_context: Mapping[str, Any] | None, + independent_retry: bool, + aggregate: bool, + ) -> _OperationState: + safe, omitted = normalize_attributes( + attributes, + self.config, + allowed_keys=( + self.config.application_attribute_keys + | self.config.dispatch_attribute_keys + ), ) - - def record_request_metric( - self, duration_seconds: float, attributes: dict[str, str] - ) -> None: - return self._metrics.record_request_metric( - duration_seconds, attributes + if omitted: + self.diagnostics.increment("attributes.omitted", omitted) + parent = None + links: list[Any] = [] + request_id: str | None = None + if remote_context: + request_id = _valid_request_id(remote_context.get("request_id")) + carrier = { + key: str(value) + for key, value in remote_context.items() + if key.lower() in {TRACEPARENT_HEADER, TRACESTATE_HEADER} + } + extracted = self._otel.extract(carrier) + direct = ( + not independent_retry + and not aggregate + and captured_at_is_recent( + remote_context.get("captured_at"), + self.config.limits.async_parent_max_age_seconds, + ) + ) + if direct: + parent = extracted + else: + link = self._otel.link(self._otel.remote_span_context(carrier)) + if link is not None: + links.append(link) + parent = self._otel.empty_context() + active_request = self._request_state.get() + if request_id is None and active_request is not None: + request_id = active_request.request_id + span = self._otel.start_span( + name, + kind=_span_kind("CONSUMER") if remote_context else None, + attributes={ + "operation.name": name, + "operation.kind": "operation", + **safe, + }, + parent_context=parent, + links=links, + ) + state = _OperationState( + name=name, + kind="operation", + request_id=request_id, + start_time=time.perf_counter(), + span=span, + attributes=safe, ) + state.token = self._operation_state.set(state) + return state - def record_segment_metric( + def _finish_operation( self, - segment: str, - duration_seconds: float, - attributes: dict[str, str], - *, - backend_segment: bool = False, + state: _OperationState | None, + error: BaseException | None, ) -> None: - return self._metrics.record_segment_metric( - segment, - duration_seconds, + if state is None or state.completed: + return + state.completed = True + state.error = error or state.error + duration = max(0.0, time.perf_counter() - state.start_time) + outcome = "error" if state.error is not None else "success" + context = { + "operation.name": state.name, + "operation.kind": state.kind, + "request.id": state.request_id, + "duration_ms": round(duration * 1_000, 3), + "outcome": outcome, + **self._otel.current_correlation(), + } + self._emit_record( + severity="ERROR" if state.error is not None else "INFO", + event_name="operation.completed", + context=context, + attributes=state.attributes, + error=state.error, + ) + metrics = { + **self._metric_base(), + "operation.name": state.name, + "operation.kind": state.kind, + "outcome": outcome, + } + self._otel.record_operation(duration, metrics) + if state.error is not None: + self._otel.record_error(metrics) + self._otel.end_span(state.span, state.error) + if state.token is not None: + try: + self._operation_state.reset(state.token) + except Exception as exc: + self.diagnostics.report("operation.context_reset", exc) + + def _start_child_span( + self, + name: str, + attributes: Mapping[str, Any] | None, + ) -> _ChildSpanState: + safe, omitted = normalize_attributes( attributes, - backend_segment=backend_segment, + self.config, + allowed_keys=( + self.config.application_attribute_keys + | self.config.dispatch_attribute_keys + ), + ) + if omitted: + self.diagnostics.increment("attributes.omitted", omitted) + return _ChildSpanState( + name=name, + start_time=time.perf_counter(), + span=self._otel.start_span(name, attributes=safe), ) - def record_error_metric(self, attributes: dict[str, str]) -> None: - return self._metrics.record_error_metric(attributes) - - def record_rate_limited_metric(self, attributes: dict[str, str]) -> None: - return self._metrics.record_rate_limited_metric(attributes) - - def record_failover_event_metric(self, attributes: dict[str, str]) -> None: - return self._metrics.record_failover_event_metric(attributes) - - def record_active_request( - self, delta: int, attributes: dict[str, str] + def _finish_child_span( + self, + state: _ChildSpanState | None, + error: BaseException | None, ) -> None: - return self._metrics.record_active_request(delta, attributes) - - def instrument_fastapi(self, app: Any) -> None: - return self._tracing.instrument_fastapi(app) - - def instrument_httpx(self) -> None: - return self._tracing.instrument_httpx() - - def shutdown(self) -> None: - budget = clamped( - self.config.shutdown_timeout_seconds, - low=0.0, - high=60.0, - default=ObservabilityConfig.shutdown_timeout_seconds, - ) - providers = [ - ("trace", self.tracer_provider), - ("metrics", self.meter_provider), - ] - providers = [ - (name, provider) - for name, provider in providers - if provider is not None - ] - # Destination close is inherently deadline-bounded, so it runs - # inline and first, with a deadline that leaves room for the - # provider flush when there is one. Everything below fits inside - # the one shutdown budget by construction. - started = time.monotonic() - try: - self.log_destination_manager.close( - budget / 2 if providers else budget - ) - except BaseException as exc: - self.log_observability_failure("logging.destination_close", exc) - if not providers: + if state is None: return - remaining = max(0.0, budget - (time.monotonic() - started)) - - def flush() -> None: - for name, provider in providers: - try: - provider.shutdown() - except BaseException as exc: - self.log_observability_failure( - f"otel.{name}_shutdown", - exc, - ) - - thread = threading.Thread( - target=flush, - name="policyengine-otel-shutdown", - daemon=True, - ) - thread.start() - thread.join(timeout=remaining) - if thread.is_alive(): - self.log_observability_failure( - "otel.shutdown_timeout", - TimeoutError("OpenTelemetry shutdown timed out."), - timeout_seconds=remaining, + self._otel.end_span(state.span, error) + + def _active_context_fields(self) -> dict[str, Any]: + fields: dict[str, Any] = {} + request = self._request_state.get() + operation = self._operation_state.get() + if request is not None: + fields.update( + { + "request.id": request.request_id, + "http.request.method": request.method, + "http.route": request.route, + } ) + if operation is not None: + fields.update( + { + "operation.name": operation.name, + "operation.kind": operation.kind, + } + ) + if operation.request_id and "request.id" not in fields: + fields["request.id"] = operation.request_id + fields.update(self._otel.current_correlation()) + return fields + + def _metric_base(self) -> dict[str, Any]: + return { + "service.name": self.config.service.name, + "service.role": self.config.service.role, + "deployment.environment.name": ( + self.config.deployment.environment + ), + "cloud.platform": self.config.deployment.platform, + } + + def _emit_record(self, **kwargs: Any) -> None: + try: + self._delivery.emit(build_record(self.config, **kwargs)) + except Exception as exc: + self.diagnostics.report("record.emit", exc) - def shutdown_tracing(self) -> None: - self.shutdown() - - def restart_log_destinations(self) -> None: - """Close and rebuild log destinations from configuration. - - Call ONLY from single-threaded lifecycle moments — a - post-snapshot-restore hook, a post-fork hook, before serving - traffic. There is deliberately no locking here: under that - contract there is no concurrency, and a violated contract costs - at most a counted drop into a closing destination. - - A no-op when observability is disabled, mirroring configure(): - the kill switch must hold across forks and snapshot restores. - """ - if not self.enabled: + def _reset_request(self, state: _RequestState) -> None: + if state.token is None: return - self.log_destination_manager.configure() + try: + self._request_state.reset(state.token) + except Exception as exc: + self.diagnostics.report("request.context_reset", exc) - def log_observability_failure( - self, operation: str, exc: BaseException, **fields: Any + def _register_logging_handler( + self, + logger: logging.Logger, + handler: ObservabilityLogHandler, ) -> None: - return self._logging.log_observability_failure( - operation, exc, **fields - ) - - def _configure_loggers(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._configure_loggers(*args, **kwargs) - - def _emit_structured_log(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._emit_structured_log(*args, **kwargs) - - def _handle_destination_failure(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._handle_destination_failure(*args, **kwargs) - - def _severity_for_log_record(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._severity_for_log_record(*args, **kwargs) - - def _int_or_none(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._int_or_none(*args, **kwargs) - - def _internal_error_payload(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._internal_error_payload(*args, **kwargs) - - def _configure_otel(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._configure_otel(*args, **kwargs) - - def _add_trace_exporter(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._add_trace_exporter(*args, **kwargs) - - def _metric_reader(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._metric_reader(*args, **kwargs) - - def _configure_instruments(self, *args: Any, **kwargs: Any) -> Any: - return self._metrics._configure_instruments(*args, **kwargs) + self._logging_handlers.append((logger, handler)) - def _instrument(self, *args: Any, **kwargs: Any) -> Any: - return self._metrics._instrument(*args, **kwargs) + def _remove_logging_handlers(self) -> None: + for logger, handler in self._logging_handlers: + try: + logger.removeHandler(handler) + except Exception as exc: + self.diagnostics.report("logging.handler_remove", exc) + self._logging_handlers.clear() - def _start_request_span(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._start_request_span(*args, **kwargs) - def _close_request_span(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._close_request_span(*args, **kwargs) - - def _safe_span(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._safe_span(*args, **kwargs) +class _ScopeManager: + def __init__( + self, + runtime: ObservabilityRuntime, + *, + scope_type: str, + name: str, + attributes: Mapping[str, Any] | None, + remote_context: Mapping[str, Any] | None = None, + independent_retry: bool = False, + aggregate: bool = False, + ) -> None: + self.runtime = runtime + self.scope_type = scope_type + self.name = str(name) + self.attributes = attributes + self.remote_context = remote_context + self.independent_retry = independent_retry + self.aggregate = aggregate + self.state: _OperationState | _ChildSpanState | None = None + + def __enter__(self) -> Any: + try: + if self.scope_type == "operation": + self.state = self.runtime._start_operation( + self.name, + self.attributes, + self.remote_context, + self.independent_retry, + self.aggregate, + ) + else: + self.state = self.runtime._start_child_span( + self.name, self.attributes + ) + except Exception as exc: + self.runtime.diagnostics.report( + f"{self.scope_type}.start", exc, name=self.name + ) + self.state = None + return self.state - def _start_span(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._start_span(*args, **kwargs) + def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> bool: + try: + if self.scope_type == "operation": + self.runtime._finish_operation(self.state, exc) # type: ignore[arg-type] + else: + self.runtime._finish_child_span(self.state, exc) # type: ignore[arg-type] + except Exception as observability_error: + self.runtime.diagnostics.report( + f"{self.scope_type}.finish", + observability_error, + name=self.name, + ) + return False - def _end_span(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._end_span(*args, **kwargs) + async def __aenter__(self) -> Any: + return self.__enter__() - def _start_segment_tree_node(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._start_segment_tree_node(*args, **kwargs) + async def __aexit__(self, exc_type: Any, exc: Any, traceback: Any) -> bool: + return self.__exit__(exc_type, exc, traceback) - def _finish_segment_tree_node(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._finish_segment_tree_node(*args, **kwargs) + def __call__(self, function: Any) -> Any: + if inspect.iscoroutinefunction(function): - def _reset_segment_tree_stack(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._reset_segment_tree_stack(*args, **kwargs) + @wraps(function) + async def async_wrapper(*args: Any, **kwargs: Any) -> Any: + async with self.runtime._scope_copy(self): + return await function(*args, **kwargs) - def _segment_tree_owner(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._segment_tree_owner(*args, **kwargs) + return async_wrapper - def _safe_segment_tree_attrs(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._safe_segment_tree_attrs(*args, **kwargs) + @wraps(function) + def wrapper(*args: Any, **kwargs: Any) -> Any: + with self.runtime._scope_copy(self): + return function(*args, **kwargs) - def _record_segment_flat_timing(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._record_segment_flat_timing(*args, **kwargs) + return wrapper - def _record_segment_safely(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._record_segment_safely(*args, **kwargs) - def _record_timing(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._record_timing(*args, **kwargs) +class ObservabilityLogHandler(logging.Handler): + _IGNORED_PREFIXES = ( + "google.", + "grpc", + "opentelemetry.", + "policyengine_observability.", + ) - def _segment_span_attributes(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._segment_span_attributes(*args, **kwargs) + def __init__(self, runtime: ObservabilityRuntime) -> None: + super().__init__(level=runtime.config.logging.minimum_severity) + self.runtime = runtime - def _span_name(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._span_name(*args, **kwargs) + def emit(self, record: logging.LogRecord) -> None: + if record.name.startswith(self._IGNORED_PREFIXES): + return + try: + attributes = getattr(record, "policyengine_attributes", None) + error = record.exc_info[1] if record.exc_info else None + self.runtime.log( + record.getMessage(), + severity=record.levelname, + attributes=attributes + if isinstance(attributes, Mapping) + else None, + error=error, + ) + except Exception as exc: + self.runtime.diagnostics.report("logging.handler_emit", exc) - def _start_implicit_operation(self, *args: Any, **kwargs: Any) -> Any: - return self._operations._start_implicit_operation(*args, **kwargs) - def _coerce_segment(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._coerce_segment(*args, **kwargs) +def instrument_logging( + logger: logging.Logger, + runtime: ObservabilityRuntime, + *, + replace: bool = False, +) -> ObservabilityLogHandler: + for handler in logger.handlers: + if ( + isinstance(handler, ObservabilityLogHandler) + and handler.runtime is runtime + ): + return handler + if replace: + logger.handlers.clear() + handler = ObservabilityLogHandler(runtime) + logger.addHandler(handler) + runtime._register_logging_handler(logger, handler) + return handler - def _set_current_span_attributes(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._set_current_span_attributes(*args, **kwargs) - def _current_span(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._current_span(*args, **kwargs) +def configure(config: ObservabilityConfig) -> ObservabilityRuntime: + return ObservabilityRuntime(config) - def _trace_ids(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._trace_ids(*args, **kwargs) - def _extract_context(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._extract_context(*args, **kwargs) +def _request_id(value: Any) -> str: + return _valid_request_id(value) or str(uuid.uuid4()) - def _record_exception_on_span(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._record_exception_on_span(*args, **kwargs) - def _add_span_event(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._add_span_event(*args, **kwargs) +def _header_value(headers: Mapping[str, str], name: str) -> str | None: + lowered = name.lower() + for key, value in headers.items(): + if key.lower() == lowered: + return value + return None - def _close_active_request(self, *args: Any, **kwargs: Any) -> Any: - return self._requests._close_active_request(*args, **kwargs) - def _reset_request_operation_context( - self, *args: Any, **kwargs: Any - ) -> Any: - return self._requests._reset_request_operation_context(*args, **kwargs) +def _valid_request_id(value: Any) -> str | None: + if isinstance(value, str) and _REQUEST_ID_PATTERN.fullmatch(value): + return value + return None - def _reset_request_context(self, *args: Any, **kwargs: Any) -> Any: - return self._requests._reset_request_context(*args, **kwargs) - def _safe_perf_counter(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._safe_perf_counter(*args, **kwargs) +def _status_class(status_code: int | None) -> str: + if status_code is None or status_code < 100 or status_code > 599: + return "unknown" + return f"{status_code // 100}xx" - def _safe_str(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._safe_str(*args, **kwargs) - def _safe_traceback(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._safe_traceback(*args, **kwargs) +def _outcome(status_code: int | None, error: BaseException | None) -> str: + if error is not None or (status_code is not None and status_code >= 500): + return "error" + if status_code is not None and status_code >= 400: + return "client_error" + return "success" - def _json(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._json(*args, **kwargs) - def _write_stderr(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._write_stderr(*args, **kwargs) +def _span_kind(name: str) -> Any: + try: + from opentelemetry.trace import SpanKind + return getattr(SpanKind, name) + except Exception: + return None -_RUNTIME = ObservabilityRuntime(ObservabilityConfig()) +def _run_bounded( + function: Any, + timeout_seconds: float, + diagnostics: Diagnostics, + diagnostic_name: str, +) -> None: + completed = threading.Event() -def set_observability_runtime(runtime: ObservabilityRuntime) -> None: - global _RUNTIME - _RUNTIME = runtime - for context_var in ( - _state._REQUEST_CONTEXT, - _state._OPERATION_CONTEXT, - _state._TIMINGS, - _state._TURN_START, - ): + def run() -> None: try: - context_var.set(None) - except BaseException: - continue - - -def observability_runtime() -> ObservabilityRuntime: - return _RUNTIME + function() + except Exception as exc: + diagnostics.report(diagnostic_name, exc) + finally: + completed.set() + + thread = threading.Thread(target=run, daemon=True) + thread.start() + completed.wait(max(0.0, timeout_seconds)) + if not completed.is_set(): + diagnostics.increment("shutdown.timeout") + diagnostics.report( + diagnostic_name, + "Operation exceeded the configured shutdown deadline.", + ) diff --git a/policyengine_observability/schema.py b/policyengine_observability/schema.py new file mode 100644 index 0000000..9e529f2 --- /dev/null +++ b/policyengine_observability/schema.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import math +import traceback +from collections.abc import Mapping +from datetime import UTC, datetime +from enum import Enum +from typing import Any + +from .config import ObservabilityConfig + +SCHEMA_VERSION = "policyengine.observability.v2" + +_PROHIBITED_KEY_PARTS = ( + "authorization", + "cookie", + "credential", + "password", + "secret", + "token", + "request_body", + "response_body", + "household", + "person", + "reform", + "payload", + "raw_ip", + "client_ip", + "prompt", + "model_response", +) + + +def normalize_attributes( + values: Mapping[str, Any] | None, + config: ObservabilityConfig, + *, + allowed_keys: frozenset[str] | None = None, +) -> tuple[dict[str, str | int | float | bool], int]: + normalized: dict[str, str | int | float | bool] = {} + omitted = 0 + for raw_key, value in (values or {}).items(): + key = str(raw_key).strip() + if ( + not key + or len(normalized) >= config.limits.max_attributes + or _prohibited_key(key) + or (allowed_keys is not None and key not in allowed_keys) + ): + omitted += 1 + continue + scalar = _normalize_scalar(value, config.limits.max_string_length) + if scalar is None: + omitted += 1 + continue + normalized[key] = scalar + return normalized, omitted + + +def build_record( + config: ObservabilityConfig, + *, + severity: str, + event_name: str | None = None, + message: str | None = None, + context: Mapping[str, Any] | None = None, + attributes: Mapping[str, Any] | None = None, + error: BaseException | None = None, +) -> dict[str, Any]: + record: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "timestamp": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + "severity": severity.upper(), + "service.name": _limit(config.service.name, config), + "service.namespace": _limit(config.service.namespace, config), + "service.version": _limit(config.service.version, config), + "service.role": _limit(config.service.role, config), + "deployment.environment.name": _limit( + config.deployment.environment, config + ), + "cloud.platform": config.deployment.platform, + } + if config.deployment.region: + record["cloud.region"] = _limit(config.deployment.region, config) + if config.deployment.instance_id: + record["service.instance.id"] = _limit( + config.deployment.instance_id, config + ) + if event_name: + record["event.name"] = _limit(event_name, config) + if message: + record["message"] = _redact(_limit(message, config), config) + + for key, value in (context or {}).items(): + if value is not None: + record[str(key)] = value + + safe_attributes, omitted = normalize_attributes( + attributes, + config, + allowed_keys=config.application_attribute_keys + | config.dispatch_attribute_keys, + ) + if safe_attributes: + record["attributes"] = safe_attributes + if omitted: + record["attributes.omitted_count"] = omitted + + if error is not None: + record.update(_error_fields(error, config)) + + trace_id = record.get("trace_id") + span_id = record.get("span_id") + project_id = config.google_cloud_project_id + if trace_id and project_id: + record["logging.googleapis.com/trace"] = ( + f"projects/{project_id}/traces/{trace_id}" + ) + record["logging.googleapis.com/trace_sampled"] = bool( + record.get("trace_sampled", False) + ) + if span_id: + record["logging.googleapis.com/spanId"] = span_id + return record + + +def metric_attributes( + values: Mapping[str, Any], config: ObservabilityConfig +) -> dict[str, str | int | float | bool]: + normalized, _ = normalize_attributes( + values, + config, + allowed_keys=config.metric_attribute_keys, + ) + return normalized + + +def _error_fields( + error: BaseException, config: ObservabilityConfig +) -> dict[str, Any]: + try: + message = str(error) + except Exception: + message = "" + try: + stack = "".join( + traceback.format_exception(type(error), error, error.__traceback__) + ) + except Exception: + stack = "" + return { + "error.type": type(error).__name__, + "error.message": _redact( + message[: config.limits.max_error_message_length], config + ), + "error.stack": _redact( + stack[: config.limits.max_stack_length], config + ), + } + + +def _normalize_scalar( + value: Any, max_length: int +) -> str | int | float | bool | None: + if value is None: + return None + if isinstance(value, Enum): + value = value.value + if isinstance(value, bool): + return value + if isinstance(value, int): + return value + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, str): + return value[:max_length] + return None + + +def _prohibited_key(key: str) -> bool: + lowered = key.lower() + return any(part in lowered for part in _PROHIBITED_KEY_PARTS) + + +def _limit(value: Any, config: ObservabilityConfig) -> str: + try: + return str(value)[: config.limits.max_string_length] + except Exception: + return "" + + +def _redact(value: str, config: ObservabilityConfig) -> str: + redacted = value + for sensitive in config.sensitive_values: + if sensitive: + redacted = redacted.replace(sensitive, "[REDACTED]") + return redacted diff --git a/policyengine_observability/segments.py b/policyengine_observability/segments.py deleted file mode 100644 index 588ed9d..0000000 --- a/policyengine_observability/segments.py +++ /dev/null @@ -1,432 +0,0 @@ -"""Segment names, nested timings, and context managers.""" - -from __future__ import annotations - -import inspect -import time -from collections.abc import AsyncIterator, Iterable, Iterator -from contextlib import asynccontextmanager, contextmanager -from enum import Enum -from functools import wraps -from typing import TYPE_CHECKING, Any - -from . import _state -from .context import ( - OperationObservabilityContext, - RequestObservabilityContext, - SegmentTimingNode, - _metric_attrs, -) - -if TYPE_CHECKING: - from .runtime import ObservabilityRuntime - -UNKNOWN_SEGMENT = "unknown_segment" - - -def segment_values( - registry: type[Enum] | Iterable[str] | None, -) -> frozenset[str]: - if registry is None: - return frozenset() - if isinstance(registry, type) and issubclass(registry, Enum): - return frozenset(str(member.value) for member in registry) - return frozenset(str(value) for value in registry) - - -def coerce_segment_name( - value: Any, - *, - registry: type[Enum] | Iterable[str] | None = None, -) -> tuple[str, bool]: - values = segment_values(registry) - if isinstance(value, Enum): - segment = str(value.value) - return segment, not values or segment in values - if isinstance(value, str): - return value, not values or value in values - try: - segment = str(value) - except BaseException: - return UNKNOWN_SEGMENT, False - return segment, False if values else True - - -MAX_SEGMENT_ATTR_LENGTH = 200 -SENSITIVE_SEGMENT_ATTR_PARTS = ( - "authorization", - "credential", - "password", - "secret", - "token", -) - - -class SegmentRecorder: - def __init__(self, runtime: ObservabilityRuntime) -> None: - self.runtime = runtime - - def segment(self, name: Any, **attrs: Any) -> Iterator[Any]: - return _SegmentManager(self.runtime, name, attrs) - - @contextmanager - def _segment_context(self, name: Any, **attrs: Any) -> Iterator[Any]: - if not self.runtime.enabled: - yield None - return - segment_name = self.runtime._coerce_segment(name) - implicit_operation = self.runtime._start_implicit_operation( - segment_name, - attrs, - ) - start = self.runtime._safe_perf_counter( - f"segment.{segment_name}.start" - ) - segment_tree_handle = self.runtime._start_segment_tree_node( - segment_name, - attrs, - ) - span_attrs = self.runtime._segment_span_attributes(attrs) - span_name = self.runtime._span_name(segment_name) - error: BaseException | None = None - with self.runtime._safe_span(span_name, span_attrs) as span: - try: - yield span - except BaseException as exc: - error = exc - self.runtime._record_segment_safely( - segment_name, - start, - attrs, - segment_tree_handle=segment_tree_handle, - ) - raise - else: - self.runtime._record_segment_safely( - segment_name, - start, - attrs, - segment_tree_handle=segment_tree_handle, - ) - finally: - self.runtime._reset_segment_tree_stack(segment_tree_handle) - self.runtime.end_operation(implicit_operation, error) - - @asynccontextmanager - async def asegment(self, name: Any, **attrs: Any) -> AsyncIterator[Any]: - if not self.runtime.enabled: - yield None - return - segment_name = self.runtime._coerce_segment(name) - implicit_operation = self.runtime._start_implicit_operation( - segment_name, - attrs, - ) - start = self.runtime._safe_perf_counter( - f"segment.{segment_name}.start" - ) - segment_tree_handle = self.runtime._start_segment_tree_node( - segment_name, - attrs, - ) - span_attrs = self.runtime._segment_span_attributes(attrs) - span_name = self.runtime._span_name(segment_name) - error: BaseException | None = None - with self.runtime._safe_span(span_name, span_attrs) as span: - try: - yield span - except BaseException as exc: - error = exc - self.runtime._record_segment_safely( - segment_name, - start, - attrs, - segment_tree_handle=segment_tree_handle, - ) - raise - else: - self.runtime._record_segment_safely( - segment_name, - start, - attrs, - segment_tree_handle=segment_tree_handle, - ) - finally: - self.runtime._reset_segment_tree_stack(segment_tree_handle) - self.runtime.end_operation(implicit_operation, error) - - def _start_segment_tree_node( - self, - name: str, - attrs: dict[str, Any], - ) -> dict[str, Any] | None: - try: - owner = self.runtime._segment_tree_owner() - if owner is None: - return None - owner.segment_sequence[0] += 1 - node = SegmentTimingNode( - sequence=owner.segment_sequence[0], - name=name, - attrs=self.runtime._safe_segment_tree_attrs(attrs), - ) - owner_id = id(owner.segment_tree) - stack = _state._SEGMENT_STACK.get() - if stack and stack[-1][0] == owner_id: - stack[-1][1].children.append(node) - else: - owner.segment_tree.append(node) - token = _state._SEGMENT_STACK.set((*stack, (owner_id, node))) - return {"node": node, "token": token} - except BaseException as exc: - self.runtime.log_observability_failure( - "segment.tree_start", - exc, - segment=name, - ) - return None - - def _finish_segment_tree_node( - self, - handle: dict[str, Any] | None, - duration_seconds: float, - ) -> None: - if not handle: - return - try: - node = handle.get("node") - if not isinstance(node, SegmentTimingNode): - return - node.duration_ms = duration_seconds * 1000 - except BaseException as exc: - self.runtime.log_observability_failure( - "segment.tree_finish", - exc, - ) - - def _reset_segment_tree_stack( - self, - handle: dict[str, Any] | None, - ) -> None: - if not handle: - return - token = handle.get("token") - if token is None: - return - try: - _state._SEGMENT_STACK.reset(token) - except BaseException as exc: - self.runtime.log_observability_failure("segment.tree_reset", exc) - - def _segment_tree_owner( - self, - ) -> RequestObservabilityContext | OperationObservabilityContext | None: - context = self.runtime.current_context() - if context is not None: - return context - return self.runtime.current_operation() - - def _safe_segment_tree_attrs( - self, - attrs: dict[str, Any], - ) -> dict[str, Any]: - safe_attrs: dict[str, Any] = {} - for key, value in attrs.items(): - key_text = self.runtime._safe_str(key) - key_lower = key_text.lower() - if any(part in key_lower for part in SENSITIVE_SEGMENT_ATTR_PARTS): - continue - if value is None: - continue - if hasattr(value, "value"): - value = value.value - if isinstance(value, bool | int | float): - safe_attrs[key_text] = value - elif isinstance(value, str): - safe_attrs[key_text] = value[:MAX_SEGMENT_ATTR_LENGTH] - return safe_attrs - - def _record_segment_flat_timing( - self, - context: RequestObservabilityContext | None, - operation: OperationObservabilityContext | None, - name: str, - duration_ms: float, - ) -> None: - seen_timing_ids: set[int] = set() - seen_count_ids: set[int] = set() - for target in (context, operation): - if target is None: - continue - timings_id = id(target.timings_ms) - if timings_id not in seen_timing_ids: - target.timings_ms[name] = round( - target.timings_ms.get(name, 0.0) + duration_ms, - 3, - ) - seen_timing_ids.add(timings_id) - counts_id = id(target.timing_counts) - if counts_id not in seen_count_ids: - target.timing_counts[name] = ( - target.timing_counts.get(name, 0) + 1 - ) - seen_count_ids.add(counts_id) - - def _record_segment_safely( - self, - name: str, - start: float | None, - attrs: dict[str, Any], - *, - segment_tree_handle: dict[str, Any] | None = None, - ) -> None: - if start is None: - return - end = self.runtime._safe_perf_counter(f"segment.{name}.end") - if end is None: - return - try: - duration = end - start - self.runtime._finish_segment_tree_node( - segment_tree_handle, duration - ) - self.runtime._record_timing(name, duration) - context = self.runtime.current_context() - operation = self.runtime.current_operation() - metric_extra = { - key: value - for key, value in attrs.items() - if ( - key in self.runtime.config.metric_attribute_keys - and value is not None - ) - } - duration_ms = duration * 1000 - self.runtime._record_segment_flat_timing( - context, - operation, - name, - duration_ms, - ) - if operation is not None: - metric_attributes = operation.metric_attributes( - segment=name, - **metric_extra, - ) - elif context is not None: - metric_attributes = context.metric_attributes( - segment=name, - **metric_extra, - ) - else: - metric_attributes = _metric_attrs( - { - "service.name": self.runtime.config.service_name, - "service.role": self.runtime.config.service_role, - "deployment.environment": self.runtime.config.environment, - "segment": name, - **metric_extra, - }, - self.runtime.config.metric_attribute_keys, - ) - self.runtime.record_segment_metric( - name, - duration, - metric_attributes, - backend_segment="backend" in metric_extra, - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "request.record_segment", - exc, - segment=name, - ) - - def _record_timing(self, name: str, duration_seconds: float) -> None: - try: - timings = _state._TIMINGS.get() - if timings is None: - return - key = f"{name}_ms" - duration_ms = duration_seconds * 1000.0 - timings[key] = round(timings.get(key, 0.0) + duration_ms, 1) - except BaseException as exc: - self.runtime.log_observability_failure( - "scope.record_timing", - exc, - segment=name, - ) - - def _coerce_segment(self, name: Any) -> str: - segment, is_registered = coerce_segment_name( - name, - registry=self.runtime.segment_registry, - ) - if not is_registered: - self.runtime.log_observability_failure( - "segment.coerce", - ValueError("Unregistered observability segment."), - segment=segment, - segment_type=type(name).__name__, - ) - return segment - - def _safe_perf_counter(self, operation: str) -> float | None: - try: - return time.perf_counter() - except BaseException as exc: - self.runtime.log_observability_failure(operation, exc) - return None - - -class _SegmentManager: - def __init__( - self, - runtime: ObservabilityRuntime, - name: Any, - attrs: dict[str, Any], - ) -> None: - self.runtime = runtime - self.name = name - self.attrs = attrs - self.context_manager = None - - def __enter__(self): - self.context_manager = self.runtime._segment_context( - self.name, - **self.attrs, - ) - return self.context_manager.__enter__() - - def __exit__(self, exc_type, exc, traceback) -> bool: - if self.context_manager is None: - return False - return bool(self.context_manager.__exit__(exc_type, exc, traceback)) - - async def __aenter__(self): - self.context_manager = self.runtime.asegment(self.name, **self.attrs) - return await self.context_manager.__aenter__() - - async def __aexit__(self, exc_type, exc, traceback) -> bool: - if self.context_manager is None: - return False - return bool( - await self.context_manager.__aexit__(exc_type, exc, traceback) - ) - - def __call__(self, func): - if inspect.iscoroutinefunction(func): - - @wraps(func) - async def async_wrapper(*args, **kwargs): - async with self.runtime.segment(self.name, **self.attrs): - return await func(*args, **kwargs) - - return async_wrapper - - @wraps(func) - def wrapper(*args, **kwargs): - with self.runtime.segment(self.name, **self.attrs): - return func(*args, **kwargs) - - return wrapper diff --git a/pyproject.toml b/pyproject.toml index 7818276..db4df3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,20 +4,24 @@ version = "1.4.1" description = "Shared PolicyEngine observability runtime for logs, timings, metrics, and OpenTelemetry." readme = "README.md" authors = [{ name = "PolicyEngine", email = "hello@policyengine.org" }] -requires-python = ">=3.12" -dependencies = [ +requires-python = ">=3.11" +dependencies = [] + +[project.optional-dependencies] +otel = [ "opentelemetry-api>=1.43.0", "opentelemetry-sdk>=1.43.0", +] +otlp-grpc = [ + "opentelemetry-api>=1.43.0", "opentelemetry-exporter-otlp-proto-grpc>=1.43.0", + "opentelemetry-sdk>=1.43.0", +] +otlp-http = [ + "opentelemetry-api>=1.43.0", "opentelemetry-exporter-otlp-proto-http>=1.43.0", - "opentelemetry-instrumentation-fastapi>=0.64b0", - "opentelemetry-instrumentation-httpx>=0.64b0", + "opentelemetry-sdk>=1.43.0", ] - -[project.optional-dependencies] -otel = [] -otlp-grpc = [] -otlp-http = [] flask = [ "flask>=2.2", ] @@ -28,18 +32,25 @@ httpx = [ "httpx", ] google = [ + "google-auth>=2.38.0", "google-cloud-logging>=3.15.0", ] all = [ "fastapi", "flask>=2.2", + "google-auth>=2.38.0", "google-cloud-logging>=3.15.0", "httpx", + "opentelemetry-api>=1.43.0", + "opentelemetry-exporter-otlp-proto-grpc>=1.43.0", + "opentelemetry-exporter-otlp-proto-http>=1.43.0", + "opentelemetry-sdk>=1.43.0", ] dev = [ "build", "coverage", "pytest", + "pyright>=1.1.405", "ruff>=0.9.0", "towncrier>=24.8.0", ] @@ -51,9 +62,19 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["policyengine_observability"] +[tool.hatch.build.targets.sdist] +exclude = [ + "/.claude", + "/.codex", + "/.pytest_cache", + "/.venv", + "/dist", + "/openspec", +] + [tool.ruff] line-length = 79 -target-version = "py312" +target-version = "py311" [tool.ruff.lint] select = [ @@ -76,6 +97,12 @@ line-ending = "auto" [tool.pytest.ini_options] testpaths = ["tests"] +[tool.pyright] +include = ["policyengine_observability"] +venvPath = "." +venv = ".venv" +pythonVersion = "3.11" + [tool.coverage.run] branch = true source = ["policyengine_observability"] diff --git a/tests/conftest.py b/tests/conftest.py index 9114b37..b20f6db 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,41 +1,57 @@ from __future__ import annotations +import io +import json +from collections.abc import Iterator + import pytest -from fakes import RecordingDestination -from policyengine_observability import _state -from policyengine_observability.destinations.registry import ( - _STRATEGIES, - register_destination, +from policyengine_observability import ( + DeploymentIdentity, + LoggingConfig, + ObservabilityConfig, + OTelConfig, + ServiceIdentity, + configure, ) +from policyengine_observability.runtime import ObservabilityRuntime + + +def make_config(**overrides): + values = { + "service": ServiceIdentity( + name="test-api", + namespace="policyengine.api-v1", + version="2.3.4", + role="entry", + ), + "deployment": DeploymentIdentity( + environment="test", + platform="local", + region="us-central1", + instance_id="instance-1", + ), + "google_cloud_project_id": "policyengine-observability", + "logging": LoggingConfig(stdout_enabled=True), + "otel": OTelConfig(enabled=False), + } + values.update(overrides) + return ObservabilityConfig(**values) + + +def make_runtime(**overrides) -> tuple[ObservabilityRuntime, io.StringIO]: + output = io.StringIO() + runtime = configure(make_config(**overrides)) + runtime._delivery._stdout = output + return runtime, output -@pytest.fixture(autouse=True) -def isolated_observability_context(): - """Keep tests independent when request and operation tests run in separate files.""" - variables = ( - (_state._REQUEST_CONTEXT, None), - (_state._OPERATION_CONTEXT, None), - (_state._TIMINGS, None), - (_state._TURN_START, None), - (_state._SEGMENT_STACK, ()), - ) - for variable, default in variables: - variable.set(default) - try: - yield - finally: - for variable, default in variables: - variable.set(default) +def records(output: io.StringIO) -> list[dict]: + return [json.loads(line) for line in output.getvalue().splitlines()] @pytest.fixture -def fake_remote_strategy(): - """Register a fake remote strategy; yields its destination name.""" - register_destination( - "fake-remote", - lambda **kwargs: RecordingDestination(), - transport="remote", - ) - yield "fake_remote" - _STRATEGIES.pop("fake_remote", None) +def runtime() -> Iterator[tuple[ObservabilityRuntime, io.StringIO]]: + value = make_runtime() + yield value + value[0].shutdown() diff --git a/tests/fakes.py b/tests/fakes.py deleted file mode 100644 index b4ea394..0000000 --- a/tests/fakes.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Shared destination fakes and manager builders for the test suite.""" - -from __future__ import annotations - -import json -import logging -import threading - -from policyengine_observability.config import ObservabilityConfig -from policyengine_observability.destinations.manager import ( - LogDestinationManager, -) - - -class RecordingLogger: - """Captures level-routed serialized lines like a stdlib logger.""" - - def __init__(self) -> None: - self.lines = [] - - def info(self, message) -> None: - self.lines.append(("INFO", message)) - - def warning(self, message) -> None: - self.lines.append(("WARNING", message)) - - def error(self, message) -> None: - self.lines.append(("ERROR", message)) - - -class RecordingDestination: - name = "recording" - - def __init__(self) -> None: - self.calls = [] - - @property - def payloads(self): - return [payload for payload, *_ in self.calls] - - def emit(self, payload, *, log_type, severity, timestamp=None) -> None: - self.calls.append((payload, log_type, severity, timestamp)) - - -class ClosableRecordingDestination(RecordingDestination): - """A recording destination with a zero-argument duck-typed close.""" - - def __init__(self) -> None: - super().__init__() - self.closed = 0 - - def close(self) -> None: - self.closed += 1 - - -class TimestampBlindDestination: - name = "timestamp-blind" - - def __init__(self) -> None: - self.calls = [] - - def emit(self, payload, *, log_type, severity) -> None: - self.calls.append((payload, log_type, severity)) - - -class BlockingDestination: - """Blocks inside emit until released; sequenced via Events.""" - - name = "blocking" - - def __init__(self) -> None: - self.entered = threading.Event() - self.release = threading.Event() - self.calls = 0 - self.closed = 0 - - def emit(self, payload, *, log_type, severity, timestamp=None) -> None: - self.calls += 1 - self.entered.set() - self.release.wait(10) - - def close(self) -> None: - self.closed += 1 - - -class FailingDestination: - """Raises from emit — always, or only for the first ``fail_first``.""" - - name = "failing" - - def __init__(self, fail_first: int | None = None) -> None: - self.calls = 0 - self.fail_first = fail_first - self.delivered = [] - - def emit(self, payload, *, log_type, severity, timestamp=None) -> None: - self.calls += 1 - if self.fail_first is None or self.calls <= self.fail_first: - raise RuntimeError("emit failed") - self.delivered.append(payload) - - -def make_manager(config=None, *, loggers=None, destinations=None): - """A manager with a failure-capturing on_failure. - - Returns ``(manager, failures)`` where each failure is the - ``(operation, exception, fields)`` triple the manager reported. - """ - failures = [] - manager = LogDestinationManager( - config=config or ObservabilityConfig(), - loggers=loggers or {"event": logging.getLogger("test-manager")}, - serializer=json.dumps, - on_failure=lambda operation, exc, **fields: failures.append( - (operation, exc, fields) - ), - ) - if destinations is not None: - manager.destinations = list(destinations) - manager.configured = True - return manager, failures diff --git a/tests/runtime_helpers.py b/tests/runtime_helpers.py deleted file mode 100644 index 5e287f5..0000000 --- a/tests/runtime_helpers.py +++ /dev/null @@ -1,198 +0,0 @@ -from __future__ import annotations - -from enum import StrEnum -from typing import Any - -from policyengine_observability import ( - ObservabilityConfig, - ObservabilityRuntime, -) - - -class SegmentName(StrEnum): - LOAD = "load" - SAVE = "save" - - -class RecordingSpan: - def __init__(self) -> None: - self.attributes = {} - self.exceptions = [] - self.events = [] - self.status = None - - def set_attribute(self, key, value) -> None: - self.attributes[key] = value - - def record_exception(self, exc) -> None: - self.exceptions.append(exc) - - def set_status(self, status) -> None: - self.status = status - - def add_event(self, event, fields) -> None: - self.events.append((event, fields)) - - def get_span_context(self): - return type( - "SpanContext", - (), - {"is_valid": False, "trace_id": 0, "span_id": 0}, - )() - - -class NamedRecordingSpan(RecordingSpan): - def __init__(self) -> None: - super().__init__() - self.names = [] - - def update_name(self, name: str) -> None: - self.names.append(name) - - -class ValidContextSpan(RecordingSpan): - def get_span_context(self): - return type( - "SpanContext", - (), - { - "is_valid": True, - "trace_id": 0x4BF92F3577B34DA6A3CE929D0E0E4736, - "span_id": 0x00F067AA0BA902B7, - }, - )() - - -class AttributeFailingSpan(RecordingSpan): - def set_attribute(self, key, value) -> None: - raise RuntimeError("attribute failed") - - -class ExceptionFailingSpan(RecordingSpan): - def record_exception(self, exc) -> None: - raise RuntimeError("record exception failed") - - -class RecordingSpanContextManager: - def __init__( - self, - span: RecordingSpan, - *, - fail_exit: bool = False, - ) -> None: - self.span = span - self.fail_exit = fail_exit - self.exited = False - - def __enter__(self): - return self.span - - def __exit__(self, *_args): - self.exited = True - if self.fail_exit: - raise RuntimeError("span exit failed") - return False - - -class RecordingTracer: - def __init__( - self, - span: RecordingSpan | None = None, - *, - fail_enter: bool = False, - fail_exit: bool = False, - ) -> None: - self.span = span or RecordingSpan() - self.fail_enter = fail_enter - self.fail_exit = fail_exit - self.calls = [] - self.last_context_manager = None - - def start_as_current_span(self, name, **kwargs): - self.calls.append((name, kwargs)) - if self.fail_enter: - raise RuntimeError("span enter failed") - self.last_context_manager = RecordingSpanContextManager( - self.span, - fail_exit=self.fail_exit, - ) - return self.last_context_manager - - -class RecordingMeter: - def __init__(self) -> None: - self.created = [] - - def create_histogram(self, name, **kwargs): - self.created.append(("histogram", name, kwargs)) - return RecordingInstrument() - - def create_counter(self, name, **kwargs): - self.created.append(("counter", name, kwargs)) - return RecordingInstrument() - - def create_up_down_counter(self, name, **kwargs): - self.created.append(("up_down_counter", name, kwargs)) - return RecordingInstrument() - - -class RecordingInstrument: - def __init__(self) -> None: - self.calls = [] - - def add(self, value, attributes=None) -> None: - self.calls.append(("add", value, attributes)) - - def record(self, value, attributes=None) -> None: - self.calls.append(("record", value, attributes)) - - -class FailingInstrument: - def add(self, *_args, **_kwargs) -> None: - raise RuntimeError("metric failed") - - def record(self, *_args, **_kwargs) -> None: - raise RuntimeError("metric failed") - - -class RecordingLogDestination: - def __init__(self, name: str = "recording") -> None: - self.name = name - self.calls = [] - - def emit( - self, - payload: dict[str, Any], - *, - log_type: str, - severity: str, - ) -> None: - self.calls.append((payload, log_type, severity)) - - -class FailingLogDestination: - name = "failing" - - def emit(self, *_args, **_kwargs) -> None: - raise RuntimeError("destination failed") - - -class RecordingPropagator: - def __init__(self) -> None: - self.extracted = None - - def inject(self, carrier) -> None: - carrier["traceparent"] = ( - "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" - ) - - def extract(self, carrier): - self.extracted = carrier - return {"parent": carrier} - - -def runtime(**kwargs) -> ObservabilityRuntime: - return ObservabilityRuntime( - ObservabilityConfig(service_name="svc", **kwargs), - segment_registry=SegmentName, - ) diff --git a/tests/test_adapters.py b/tests/test_adapters.py new file mode 100644 index 0000000..d99a3e2 --- /dev/null +++ b/tests/test_adapters.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +import asyncio + +import httpx +from conftest import make_runtime, records +from fastapi import FastAPI +from fastapi.testclient import TestClient +from flask import Flask + +from policyengine_observability import ( + REQUEST_ID_HEADER, + instrument_fastapi, + instrument_flask, + instrument_httpx, +) + + +def test_flask_lifecycle_and_idempotence() -> None: + runtime, output = make_runtime() + unused, _ = make_runtime() + app = Flask(__name__) + + @app.get("/items/") + def item(item_id: str): + return {"item": item_id} + + assert instrument_flask(app, runtime) is runtime + assert instrument_flask(app, unused) is runtime + response = app.test_client().get( + "/items/abc", headers={REQUEST_ID_HEADER: "flask-request"} + ) + assert response.status_code == 200 + assert response.headers[REQUEST_ID_HEADER] == "flask-request" + emitted = records(output) + assert len(emitted) == 1 + assert emitted[0]["http.route"] == "/items/" + runtime.shutdown() + unused.shutdown() + + +def test_flask_exception_emits_one_error_completion() -> None: + runtime, output = make_runtime() + app = Flask(__name__) + + @app.get("/fail") + def fail(): + raise ValueError("route failed") + + instrument_flask(app, runtime) + response = app.test_client().get("/fail") + assert response.status_code == 500 + emitted = records(output) + assert len(emitted) == 1 + assert emitted[0]["outcome"] == "error" + runtime.shutdown() + + +def test_fastapi_lifecycle_and_idempotence() -> None: + runtime, output = make_runtime() + unused, _ = make_runtime() + app = FastAPI() + + @app.get("/items/{item_id}") + async def item(item_id: str): + return {"item": item_id} + + assert instrument_fastapi(app, runtime) is runtime + assert instrument_fastapi(app, unused) is runtime + with TestClient(app) as client: + response = client.get( + "/items/abc", headers={REQUEST_ID_HEADER: "fastapi-request"} + ) + assert response.status_code == 200 + assert response.headers[REQUEST_ID_HEADER] == "fastapi-request" + emitted = records(output) + assert len(emitted) == 1 + assert emitted[0]["http.route"] == "/items/{item_id}" + runtime.shutdown() + unused.shutdown() + + +def test_fastapi_error_and_non_http_scope() -> None: + runtime, output = make_runtime() + app = FastAPI() + + @app.get("/fail") + async def fail(): + raise ValueError("route failed") + + instrument_fastapi(app, runtime) + with TestClient(app, raise_server_exceptions=False) as client: + response = client.get("/fail") + assert response.status_code == 500 + emitted = records(output) + assert len(emitted) == 1 + assert emitted[0]["outcome"] == "error" + runtime.shutdown() + + +def test_httpx_only_instruments_supplied_sync_client() -> None: + runtime, _output = make_runtime() + seen: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(200) + + supplied = httpx.Client(transport=httpx.MockTransport(respond)) + untouched = httpx.Client(transport=httpx.MockTransport(respond)) + assert instrument_httpx(supplied, runtime) is supplied + instrument_httpx(supplied, runtime) + runtime.begin_request( + headers={REQUEST_ID_HEADER: "outbound-request"}, + method="GET", + route="/dispatch", + ) + supplied.get("https://example.test/supplied") + untouched.get("https://example.test/untouched") + assert seen[0].headers[REQUEST_ID_HEADER] == "outbound-request" + assert REQUEST_ID_HEADER not in seen[1].headers + supplied.close() + untouched.close() + runtime.end_request(status_code=200) + runtime.shutdown() + + +def test_httpx_async_client_and_invalid_client_are_nonfatal() -> None: + runtime, _output = make_runtime() + seen: list[httpx.Request] = [] + + async def respond(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(200) + + async def run() -> None: + client = httpx.AsyncClient(transport=httpx.MockTransport(respond)) + instrument_httpx(client, runtime) + runtime.begin_request( + headers={REQUEST_ID_HEADER: "async-request"}, + method="POST", + route="/dispatch", + ) + await client.get("https://example.test/async") + await client.aclose() + runtime.end_request(status_code=200) + + asyncio.run(run()) + assert seen[0].headers[REQUEST_ID_HEADER] == "async-request" + marker = object() + assert instrument_httpx(marker, runtime) is marker + assert runtime.diagnostics.count("failure.httpx.instrument") == 1 + runtime.shutdown() + + +def test_httpx_client_cannot_be_rebound_to_another_runtime() -> None: + first, _ = make_runtime() + second, _ = make_runtime() + client = httpx.Client( + transport=httpx.MockTransport(lambda _: httpx.Response(200)) + ) + instrument_httpx(client, first) + instrument_httpx(client, second) + assert second.diagnostics.count("failure.httpx.already_instrumented") == 1 + client.close() + first.shutdown() + second.shutdown() diff --git a/tests/test_config_schema.py b/tests/test_config_schema.py new file mode 100644 index 0000000..8d7557f --- /dev/null +++ b/tests/test_config_schema.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import json + +from conftest import make_config + +from policyengine_observability import ( + SCHEMA_VERSION, + DeploymentIdentity, + GoogleCloudLoggingConfig, + LoggingConfig, + ObservabilityConfig, + OTelConfig, + ServiceIdentity, + TelemetryLimits, +) +from policyengine_observability.schema import ( + build_record, + metric_attributes, + normalize_attributes, +) + + +def test_explicit_identity_is_required_by_constructor() -> None: + config = make_config() + assert config.identity_complete + assert config.service.namespace == "policyengine.api-v1" + assert config.deployment.platform == "local" + + +def test_incomplete_identity_reports_diagnostics() -> None: + config = make_config( + service=ServiceIdentity("", "", "", ""), + deployment=DeploymentIdentity("", "other"), + logging=LoggingConfig( + remote=GoogleCloudLoggingConfig(project_id="", log_name="") + ), + otel=OTelConfig(enabled=True, endpoint=None), + ) + messages = " ".join(config.diagnostics()) + assert not config.identity_complete + assert "Missing explicit runtime identity" in messages + assert "Remote logging is disabled" in messages + assert "no OTLP endpoint" in messages + + +def test_remote_logging_is_rejected_outside_modal() -> None: + config = make_config( + logging=LoggingConfig( + remote=GoogleCloudLoggingConfig(project_id="central") + ) + ) + assert "outside Modal" in " ".join(config.diagnostics()) + + +def test_from_env_reads_transport_but_not_identity(monkeypatch) -> None: + monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "ambient-project") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://collector") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "x-one=1,bad,x-two=2") + monkeypatch.setenv("OTEL_TRACES_SAMPLER_ARG", "0.25") + monkeypatch.setenv("OTEL_BSP_MAX_QUEUE_SIZE", "12") + monkeypatch.setenv("OTEL_BSP_MAX_EXPORT_BATCH_SIZE", "99") + monkeypatch.setenv("OTEL_BSP_SCHEDULE_DELAY", "2500") + monkeypatch.setenv("OTEL_BSP_EXPORT_TIMEOUT", "1500") + monkeypatch.setenv("OTEL_METRIC_EXPORT_INTERVAL", "4000") + service = ServiceIdentity("svc", "ns", "1", "api") + deployment = DeploymentIdentity("prod", "google_cloud_run") + config = ObservabilityConfig.from_env( + service=service, deployment=deployment + ) + assert config.service is service + assert config.deployment is deployment + assert config.google_cloud_project_id is None + assert config.logging.remote is None + assert config.otel.endpoint == "https://collector" + assert config.otel.protocol == "http/protobuf" + assert config.otel.headers == (("x-one", "1"), ("x-two", "2")) + assert config.otel.sampling_ratio == 0.25 + assert config.otel.span_queue_capacity == 12 + assert config.otel.span_batch_size == 99 + assert config.otel.span_schedule_delay_seconds == 2.5 + assert config.otel.export_timeout_seconds == 1.5 + assert config.otel.metric_export_interval_seconds == 4.0 + + +def test_from_env_invalid_values_are_bounded(monkeypatch) -> None: + monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "invalid") + monkeypatch.setenv("OTEL_TRACES_SAMPLER_ARG", "nan") + monkeypatch.setenv("OTEL_BSP_MAX_QUEUE_SIZE", "bad") + monkeypatch.setenv("OTEL_SDK_DISABLED", "yes") + config = ObservabilityConfig.from_env( + service=ServiceIdentity("svc", "ns", "1", "api"), + deployment=DeploymentIdentity("dev", "local"), + ) + assert config.otel.protocol == "grpc" + assert config.otel.sampling_ratio == 1.0 + assert config.otel.span_queue_capacity == 2_048 + assert not config.otel.enabled + + +def test_schema_preserves_core_fields_and_namespaces_attributes() -> None: + config = make_config( + application_attribute_keys=frozenset({"service.name", "backend"}) + ) + record = build_record( + config, + severity="info", + event_name="work.started", + context={"request.id": "request-1"}, + attributes={"service.name": "attacker", "backend": "modal"}, + ) + assert record["schema_version"] == SCHEMA_VERSION + assert record["service.name"] == "test-api" + assert record["attributes"]["service.name"] == "attacker" + assert record["attributes"]["backend"] == "modal" + json.dumps(record) + + +def test_schema_redacts_and_truncates_errors() -> None: + config = make_config( + sensitive_values=("secret-value",), + limits=TelemetryLimits( + max_string_length=8, + max_error_message_length=64, + max_stack_length=100, + ), + ) + try: + raise ValueError("secret-value should disappear") + except ValueError as error: + record = build_record( + config, + severity="ERROR", + message="secret-value message", + error=error, + ) + assert "secret-value" not in str(record) + assert "[REDACTED]" in str(record) + assert len(record["error.stack"]) <= 100 + + +def test_attribute_policy_omits_sensitive_non_scalar_and_nonfinite() -> None: + config = make_config( + application_attribute_keys=frozenset( + {"allowed", "authorization", "items", "infinite", "flag"} + ) + ) + safe, omitted = normalize_attributes( + { + "allowed": "value", + "authorization": "bearer", + "items": [1, 2], + "infinite": float("inf"), + "flag": True, + }, + config, + allowed_keys=config.application_attribute_keys, + ) + assert safe == {"allowed": "value", "flag": True} + assert omitted == 3 + + +def test_attribute_count_and_string_length_are_bounded() -> None: + config = make_config( + application_attribute_keys=frozenset({"one", "two", "three"}), + limits=TelemetryLimits(max_attributes=2, max_string_length=3), + ) + safe, omitted = normalize_attributes( + {"one": "abcdef", "two": 2, "three": 3}, config + ) + assert safe == {"one": "abc", "two": 2} + assert omitted == 1 + + +def test_google_trace_correlation_uses_central_project() -> None: + record = build_record( + make_config(), + severity="INFO", + event_name="correlated", + context={ + "trace_id": "a" * 32, + "span_id": "b" * 16, + "trace_sampled": True, + }, + ) + assert record["logging.googleapis.com/trace"] == ( + "projects/policyengine-observability/traces/" + "a" * 32 + ) + assert record["logging.googleapis.com/spanId"] == "b" * 16 + assert record["logging.googleapis.com/trace_sampled"] is True + + +def test_metric_attributes_use_separate_allowlist() -> None: + config = make_config() + assert metric_attributes( + {"service.name": "test-api", "job_id": "high-cardinality"}, config + ) == {"service.name": "test-api"} diff --git a/tests/test_delivery.py b/tests/test_delivery.py new file mode 100644 index 0000000..294ce06 --- /dev/null +++ b/tests/test_delivery.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +import io +import json +import threading +import time + +from conftest import make_config + +from policyengine_observability import ( + DeploymentIdentity, + GoogleCloudLoggingConfig, + LoggingConfig, +) +from policyengine_observability.delivery import ( + DeliveryManager, + _GoogleCloudWriter, +) +from policyengine_observability.diagnostics import Diagnostics + + +class RecordingWriter: + def __init__(self) -> None: + self.records: list[dict] = [] + self.closed = False + self.written = threading.Event() + + def write(self, record: dict) -> None: + self.records.append(record) + self.written.set() + + def write_many(self, records: list[dict]) -> None: + self.records.extend(records) + self.written.set() + + def close(self) -> None: + self.closed = True + + +def modal_config(**remote_overrides): + remote_values = { + "project_id": "policyengine-observability", + "queue_capacity": 10, + "batch_size": 5, + "close_timeout_seconds": 0.2, + } + remote_values.update(remote_overrides) + return make_config( + deployment=DeploymentIdentity("test", "modal"), + logging=LoggingConfig( + stdout_enabled=True, + remote=GoogleCloudLoggingConfig(**remote_values), + ), + ) + + +def test_stdout_delivery_is_single_line_json() -> None: + output = io.StringIO() + manager = DeliveryManager(make_config(), Diagnostics(), stdout=output) + manager.emit({"severity": "INFO", "event.name": "test"}) + assert output.getvalue().count("\n") == 1 + assert json.loads(output.getvalue())["event.name"] == "test" + assert not manager.remote_enabled + + +def test_modal_delivery_writes_stdout_and_uses_lazy_worker() -> None: + output = io.StringIO() + diagnostics = Diagnostics() + writer = RecordingWriter() + factory_calls = 0 + + def factory(_config): + nonlocal factory_calls + factory_calls += 1 + return writer + + manager = DeliveryManager( + modal_config(), + diagnostics, + stdout=output, + writer_factory=factory, + ) + assert factory_calls == 0 + manager.emit({"severity": "INFO", "event.name": "modal"}) + assert json.loads(output.getvalue())["event.name"] == "modal" + assert writer.written.wait(1) + assert writer.records[0]["event.name"] == "modal" + assert factory_calls == 1 + manager.close(1) + assert writer.closed + + +def test_cloud_run_never_creates_direct_writer() -> None: + calls = 0 + + def factory(_config): + nonlocal calls + calls += 1 + return RecordingWriter() + + manager = DeliveryManager( + make_config( + deployment=DeploymentIdentity("prod", "google_cloud_run"), + logging=LoggingConfig( + remote=GoogleCloudLoggingConfig(project_id="central") + ), + ), + Diagnostics(), + stdout=io.StringIO(), + writer_factory=factory, + ) + manager.emit({"event.name": "cloud-run"}) + manager.close() + assert calls == 0 + assert not manager.remote_enabled + + +def test_queue_saturation_drops_without_blocking() -> None: + started = threading.Event() + release = threading.Event() + diagnostics = Diagnostics() + + class BlockingWriter(RecordingWriter): + def write_many(self, records: list[dict]) -> None: + started.set() + release.wait(1) + super().write_many(records) + + writer = BlockingWriter() + manager = DeliveryManager( + modal_config(queue_capacity=1, batch_size=1), + diagnostics, + stdout=io.StringIO(), + writer_factory=lambda _config: writer, + ) + manager.emit({"sequence": 1}) + assert started.wait(1) + manager.emit({"sequence": 2}) + before = time.perf_counter() + manager.emit({"sequence": 3}) + assert time.perf_counter() - before < 0.1 + assert diagnostics.count("logs.dropped.queue_full") == 1 + release.set() + manager.close(1) + + +def test_writer_auth_or_permission_failure_is_nonfatal() -> None: + diagnostics = Diagnostics() + attempted = threading.Event() + + def unavailable(_config): + attempted.set() + raise PermissionError("denied") + + manager = DeliveryManager( + modal_config(), + diagnostics, + stdout=io.StringIO(), + writer_factory=unavailable, + ) + manager.emit({"event.name": "survives"}) + assert attempted.wait(1) + deadline = time.time() + 1 + while diagnostics.count("logs.export_failure") == 0: + assert time.time() < deadline + time.sleep(0.01) + manager.close(1) + assert diagnostics.count("logs.export_failure") == 1 + + +def test_malformed_record_and_closed_queue_are_nonfatal() -> None: + diagnostics = Diagnostics() + output = io.StringIO() + manager = DeliveryManager(make_config(), diagnostics, stdout=output) + manager.emit({"bad": object()}) + assert diagnostics.count("failure.stdout.write") == 1 + + remote = DeliveryManager( + modal_config(), + diagnostics, + stdout=io.StringIO(), + writer_factory=lambda _config: RecordingWriter(), + ) + remote.close(1) + remote.emit({"event.name": "after-close"}) + assert diagnostics.count("logs.dropped.closed") == 1 + + +def test_shutdown_timeout_is_bounded_and_repeatable() -> None: + started = threading.Event() + release = threading.Event() + diagnostics = Diagnostics() + + class BlockingWriter(RecordingWriter): + def write_many(self, records: list[dict]) -> None: + started.set() + release.wait(1) + + manager = DeliveryManager( + modal_config(close_timeout_seconds=0.01), + diagnostics, + stdout=io.StringIO(), + writer_factory=lambda _config: BlockingWriter(), + ) + manager.emit({"event.name": "block"}) + assert started.wait(1) + before = time.perf_counter() + manager.close(0.01) + manager.close(0.01) + assert time.perf_counter() - before < 0.2 + assert diagnostics.count("logs.shutdown_timeout") == 1 + release.set() + + +def test_google_writer_batches_with_bounded_api_calls(monkeypatch) -> None: + from google.cloud import logging_v2 + + committed: list[dict] = [] + + class Batch: + def log_struct(self, record, **kwargs): + committed.append({"record": record, "kwargs": kwargs}) + + def commit(self): + committed.append({"committed": True}) + + class Logger: + def batch(self): + return Batch() + + class Gapic: + def write_log_entries(self, *args, **kwargs): + return None + + class Client: + def __init__(self, **kwargs): + self.kwargs = kwargs + self.logging_api = type("API", (), {"_gapic_api": Gapic()})() + self.closed = False + + def logger(self, name): + assert name == "policyengine-api-v1-modal" + return Logger() + + def close(self): + self.closed = True + + monkeypatch.setattr(logging_v2, "Client", Client) + monkeypatch.setattr( + "policyengine_observability.google_credentials.load_google_credentials", + lambda **_kwargs: "credentials", + ) + writer = _GoogleCloudWriter( + GoogleCloudLoggingConfig(project_id="central", write_timeout_seconds=2) + ) + writer.write_many( + [ + { + "severity": "WARNING", + "logging.googleapis.com/trace": "projects/central/traces/a", + "logging.googleapis.com/spanId": "b", + "logging.googleapis.com/trace_sampled": True, + }, + {"severity": "INFO"}, + ] + ) + assert committed[-1] == {"committed": True} + assert committed[0]["kwargs"] == { + "severity": "WARNING", + "trace": "projects/central/traces/a", + "span_id": "b", + "trace_sampled": True, + } + assert committed[1]["kwargs"]["trace_sampled"] is False + writer.close() + assert writer._client.closed diff --git a/tests/test_deployment_assets.py b/tests/test_deployment_assets.py new file mode 100644 index 0000000..791d4d6 --- /dev/null +++ b/tests/test_deployment_assets.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +ROOT = Path(__file__).parents[1] +DEPLOY = ROOT / "deploy" / "gcp" + + +def test_dashboard_is_valid_json_with_required_signals() -> None: + dashboard = json.loads((DEPLOY / "dashboard.json").read_text()) + serialized = json.dumps(dashboard) + assert "policyengine.request.count" in serialized + assert "policyengine.request.duration" in serialized + assert "policyengine.error.count" in serialized + assert "policyengine.telemetry.dropped" in serialized + assert "policyengine.telemetry.exporter.failure" in serialized + + +def test_collector_accepts_only_traces_and_metrics() -> None: + config = (DEPLOY / "collector" / "config.yaml").read_text() + assert "telemetry.googleapis.com:443" in config + assert "memory_limiter" in config + assert "googleclientauth" in config + assert " traces:" in config + assert " metrics:" in config + assert " logs:\n receivers:" not in config + + +def test_authorization_assets_exclude_unrelated_applications() -> None: + iam = (DEPLOY / "iam.yaml").read_text() + routing = (DEPLOY / "log-routing.yaml").read_text() + for excluded in ( + "policyengine-household-api", + "policyengine-uk-chat", + "peukchat", + "precompute", + "smoke", + "ephemeral", + ): + assert excluded not in iam + assert excluded not in routing + assert "policyengine-simulation-gateway" in iam + assert "policyengine-simulation-py" in iam + assert 'jsonPayload."service.namespace"' in routing + + +def test_verification_script_has_valid_shell_syntax() -> None: + subprocess.run( + ["bash", "-n", str(DEPLOY / "verify.sh")], + check=True, + capture_output=True, + text=True, + ) diff --git a/tests/test_diagnostics_public_api.py b/tests/test_diagnostics_public_api.py new file mode 100644 index 0000000..19e67e0 --- /dev/null +++ b/tests/test_diagnostics_public_api.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import ast +import io +import json +import logging +import sys + +from conftest import make_config, make_runtime, records + +import policyengine_observability as observability +from policyengine_observability.diagnostics import Diagnostics + +REMOVED_NAMES = { + "segment", + "asegment", + "entrypoint", + "set_attribute", + "record_event", + "record_error", + "collect_timings", + "start_scope", + "annotate", + "end_scope", + "set_observability_runtime", + "register_destination", +} + + +def test_public_api_contains_only_version_two_surface() -> None: + assert REMOVED_NAMES.isdisjoint(observability.__all__) + assert all(not hasattr(observability, name) for name in REMOVED_NAMES) + assert ( + observability.configure(make_config()).config.service.name + == "test-api" + ) + + +def test_import_does_not_require_framework_google_or_otel_sdk_modules() -> ( + None +): + package_source = open(observability.__file__).read() + tree = ast.parse(package_source) + top_level_modules = { + alias.name.split(".")[0] + for node in tree.body + if isinstance(node, ast.Import) + for alias in node.names + } + assert top_level_modules.isdisjoint( + {"google", "opentelemetry", "flask", "fastapi", "httpx"} + ) + + +def test_diagnostics_are_local_rate_limited_and_counted() -> None: + output = io.StringIO() + diagnostics = Diagnostics(stderr=output, interval_seconds=60) + diagnostics.report("export.failed", ValueError("one"), attempt=1) + diagnostics.report("export.failed", ValueError("two"), attempt=2) + diagnostics.increment("dropped", 3) + lines = output.getvalue().splitlines() + assert len(lines) == 1 + item = json.loads(lines[0]) + assert item["operation"] == "export.failed" + assert item["attempt"] == 1 + assert diagnostics.count("failure.export.failed") == 2 + assert diagnostics.count("dropped") == 3 + + +def test_diagnostic_listener_failure_does_not_escape() -> None: + diagnostics = Diagnostics(stderr=io.StringIO()) + diagnostics.add_listener( + lambda _name, _value: (_ for _ in ()).throw(RuntimeError("listener")) + ) + diagnostics.increment("still-counted", 2) + assert diagnostics.count("still-counted") == 2 + + +def test_standard_logging_ignores_observability_dependencies() -> None: + runtime, output = make_runtime() + logger = logging.getLogger("opentelemetry.exporter") + logger.handlers.clear() + logger.propagate = False + observability.instrument_logging(logger, runtime) + logger.error("would recurse") + assert records(output) == [] + runtime.shutdown() + + +def test_automatic_standard_logging_installation(monkeypatch) -> None: + logger = logging.getLogger() + original_handlers = list(logger.handlers) + try: + config = make_config( + logging=observability.LoggingConfig( + stdout_enabled=True, + capture_standard_library=True, + replace_existing_handlers=True, + ) + ) + runtime = observability.configure(config) + runtime._delivery._stdout = io.StringIO() + logging.getLogger("application.auto").warning("automatic") + assert records(runtime._delivery._stdout)[0]["message"] == "automatic" + runtime.shutdown() + finally: + logger.handlers[:] = original_handlers + + +def test_replace_logging_handlers_and_shutdown_removes_owned_handler() -> None: + runtime, _ = make_runtime() + logger = logging.getLogger("tests.replace") + logger.handlers.clear() + logger.addHandler(logging.StreamHandler(sys.stderr)) + handler = observability.instrument_logging(logger, runtime, replace=True) + assert logger.handlers == [handler] + runtime.shutdown() + assert logger.handlers == [] diff --git a/tests/test_fastapi_adapter.py b/tests/test_fastapi_adapter.py deleted file mode 100644 index 1ff9055..0000000 --- a/tests/test_fastapi_adapter.py +++ /dev/null @@ -1,380 +0,0 @@ -from __future__ import annotations - -import asyncio -from enum import StrEnum - -import pytest - -from policyengine_observability import ( - ObservabilityConfig, - ObservabilityRuntime, -) -from policyengine_observability.adapters.fastapi import ( - UNMATCHED_ROUTE, - FastAPIObservabilityAdapter, - FastAPIObservabilityMiddleware, - _endpoint_from_scope, - _headers_from_scope, - _int_header, - _merge_response_headers, - _query_keys, - _route_from_scope, - _split_forwarded_for, - init_fastapi_observability, -) - - -class SegmentName(StrEnum): - LOAD = "load" - - -class RecordingInstrument: - def __init__(self) -> None: - self.calls = [] - - def add(self, value, attributes=None) -> None: - self.calls.append(("add", value, attributes)) - - def record(self, value, attributes=None) -> None: - self.calls.append(("record", value, attributes)) - - -def _fastapi_modules(): - fastapi = pytest.importorskip("fastapi") - responses = pytest.importorskip("starlette.responses") - return fastapi, responses - - -def _call_asgi(app, path: str): - async def run(): - messages = [] - received_request = False - scope = { - "type": "http", - "asgi": {"version": "3.0", "spec_version": "2.4"}, - "http_version": "1.1", - "method": "GET", - "scheme": "http", - "path": path, - "raw_path": path.encode(), - "query_string": b"", - "headers": [(b"host", b"testserver")], - "client": ("127.0.0.1", 12345), - "server": ("testserver", 80), - "root_path": "", - } - - async def receive(): - nonlocal received_request - if not received_request: - received_request = True - return { - "type": "http.request", - "body": b"", - "more_body": False, - } - await asyncio.sleep(60) - return {"type": "http.disconnect"} - - async def send(message): - messages.append(message) - - await app(scope, receive, send) - return messages - - return asyncio.run(run()) - - -def test_fastapi_streaming_request_finishes_after_final_body() -> None: - fastapi, responses = _fastapi_modules() - app = fastapi.FastAPI() - observed = ObservabilityRuntime( - ObservabilityConfig( - service_name="svc", - otel_enabled=False, - metric_attribute_keys=( - "service.name", - "route", - "method", - "segment", - "tool", - ), - ), - segment_registry=SegmentName, - ) - observed.requests = RecordingInstrument() - observed.http_duration = RecordingInstrument() - observed.segment_duration = RecordingInstrument() - observed.active_requests = RecordingInstrument() - - @app.get("/items/{item_id}") - async def get_item(item_id: str): - async def body(): - with observed.segment(SegmentName.LOAD, tool="stream"): - yield f"item:{item_id}".encode() - - return responses.StreamingResponse(body()) - - init_fastapi_observability( - app, - runtime=observed, - service_name="svc", - ) - - messages = _call_asgi(app, "/items/abc") - response_start = next( - message - for message in messages - if message["type"] == "http.response.start" - ) - body = b"".join( - message.get("body", b"") - for message in messages - if message["type"] == "http.response.body" - ) - response_headers = dict(response_start["headers"]) - - assert body == b"item:abc" - assert response_headers[b"X-PolicyEngine-Request-Id"] - _, _, request_attributes = observed.requests.calls[0] - _, _, segment_attributes = observed.segment_duration.calls[0] - assert request_attributes["route"] == "/items/{item_id}" - assert segment_attributes["route"] == "/items/{item_id}" - assert segment_attributes["tool"] == "stream" - assert observed.active_requests.calls[0][1] == 1 - assert observed.active_requests.calls[-1][1] == -1 - - -def test_fastapi_unmatched_route_uses_stable_metric_label() -> None: - fastapi, _responses = _fastapi_modules() - app = fastapi.FastAPI() - observed = ObservabilityRuntime( - ObservabilityConfig(service_name="svc", otel_enabled=False) - ) - observed.requests = RecordingInstrument() - observed.http_duration = RecordingInstrument() - observed.active_requests = RecordingInstrument() - - init_fastapi_observability( - app, - runtime=observed, - service_name="svc", - ) - - messages = _call_asgi(app, "/missing/abc") - response_start = next( - message - for message in messages - if message["type"] == "http.response.start" - ) - - assert response_start["status"] == 404 - _, _, request_attributes = observed.requests.calls[0] - assert request_attributes["route"] == UNMATCHED_ROUTE - - -def test_fastapi_static_attributes_are_recorded() -> None: - fastapi, _responses = _fastapi_modules() - app = fastapi.FastAPI() - observed = ObservabilityRuntime( - ObservabilityConfig( - service_name="svc", - otel_enabled=False, - metric_attribute_keys=( - "service.name", - "route", - "method", - "platform", - "runtime_role", - ), - ) - ) - observed.requests = RecordingInstrument() - observed.http_duration = RecordingInstrument() - observed.active_requests = RecordingInstrument() - - @app.get("/ok") - async def ok(): - return {"ok": True} - - init_fastapi_observability( - app, - runtime=observed, - service_name="svc", - static_attributes={ - "platform": "modal", - "runtime_role": "modal_web", - "ignored": None, - }, - ) - - _call_asgi(app, "/ok") - - _, _, request_attributes = observed.requests.calls[0] - assert request_attributes["platform"] == "modal" - assert request_attributes["runtime_role"] == "modal_web" - assert "ignored" not in request_attributes - - -def test_fastapi_adapter_disabled_and_idempotent_paths() -> None: - fastapi, _responses = _fastapi_modules() - app = fastapi.FastAPI() - disabled = ObservabilityRuntime.disabled() - adapter = FastAPIObservabilityAdapter(disabled) - - adapter.instrument_app(app) - - assert not hasattr(app.state, "policyengine_observability_adapter") - - observed = ObservabilityRuntime( - ObservabilityConfig(service_name="svc", instrument_fastapi=True) - ) - adapter = FastAPIObservabilityAdapter(observed) - adapter.instrument_app(app) - adapter.instrument_app(app) - - assert app.state.policyengine_observability_adapter is adapter - - -def test_fastapi_adapter_logs_middleware_and_start_failures() -> None: - class BrokenApp: - state = type("State", (), {})() - - def add_middleware(self, *_args, **_kwargs): - raise RuntimeError("middleware failed") - - observed = ObservabilityRuntime(ObservabilityConfig(service_name="svc")) - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - adapter = FastAPIObservabilityAdapter(observed) - adapter.instrument_app(BrokenApp()) - observed.begin_request = lambda *_args, **_kwargs: (_ for _ in ()).throw( - RuntimeError("begin failed") - ) - adapter.start_request({"headers": [], "path": "/broken"}) - - assert failures == [ - "fastapi.middleware_install", - "fastapi.before_request", - ] - - -def test_fastapi_init_returns_existing_runtime() -> None: - fastapi, _responses = _fastapi_modules() - app = fastapi.FastAPI() - runtime = ObservabilityRuntime.disabled() - app.state.policyengine_observability = runtime - - assert init_fastapi_observability(app, service_name="svc") is runtime - - -def test_fastapi_inbound_metadata_ip_source_variants() -> None: - observed = ObservabilityRuntime( - ObservabilityConfig(service_name="svc", log_raw_ip=False) - ) - adapter = FastAPIObservabilityAdapter(observed) - - real_ip = adapter._inbound_metadata( - {"client": ("10.0.0.1", 123)}, - {"x-real-ip": "198.51.100.5"}, - ) - remote_addr = adapter._inbound_metadata( - {"client": ("10.0.0.1", 123)}, - {}, - ) - - assert real_ip["ip_source"] == "x_real_ip" - assert remote_addr["ip_source"] == "remote_addr" - assert "client_ip" not in real_ip - - -def test_fastapi_middleware_non_http_scope_passthrough() -> None: - calls = [] - - async def app(scope, _receive, _send): - calls.append(scope["type"]) - - adapter = FastAPIObservabilityAdapter( - ObservabilityRuntime(ObservabilityConfig(service_name="svc")) - ) - middleware = FastAPIObservabilityMiddleware(app, adapter=adapter) - - async def run(): - await middleware({"type": "lifespan"}, None, None) - - asyncio.run(run()) - - assert calls == ["lifespan"] - - -def test_fastapi_middleware_records_exception_before_reraising() -> None: - observed = ObservabilityRuntime(ObservabilityConfig(service_name="svc")) - observed.errors = RecordingInstrument() - observed.requests = RecordingInstrument() - observed.http_duration = RecordingInstrument() - observed.active_requests = RecordingInstrument() - - async def app(_scope, _receive, _send): - raise RuntimeError("app failed") - - adapter = FastAPIObservabilityAdapter(observed) - middleware = FastAPIObservabilityMiddleware(app, adapter=adapter) - - async def run(): - async def receive(): - return {"type": "http.request", "body": b"", "more_body": False} - - async def send(_message): - return None - - scope = { - "type": "http", - "method": "GET", - "path": "/boom", - "query_string": b"a=1", - "headers": [], - "client": ("127.0.0.1", 123), - } - with pytest.raises(RuntimeError, match="app failed"): - await middleware(scope, receive, send) - - asyncio.run(run()) - - assert observed.errors.calls[0][0] == "add" - assert observed.requests.calls[0][0] == "add" - - -def test_fastapi_helpers_handle_edge_inputs() -> None: - scope = { - "headers": [ - (b"x-test", b"one"), - (b"x-test", b"two"), - (object(), object()), - ], - "query_string": "a=1&b=&a=2", - } - - assert _headers_from_scope(scope)["x-test"] == "one,two" - assert _query_keys(scope) == ["a", "b"] - assert _route_from_scope({"route": object()}) is None - assert _endpoint_from_scope({"endpoint": "callable-ish"}) == "callable-ish" - assert _int_header("bad") is None - assert _int_header("12") == 12 - assert _split_forwarded_for("1.1.1.1, ,2.2.2.2") == [ - "1.1.1.1", - "2.2.2.2", - ] - assert _merge_response_headers( - [(b"x-policyengine-request-id", b"old"), (b"x-other", b"keep")], - { - "X-PolicyEngine-Request-Id": "new", - "traceparent": "parent", - "ignored": "value", - }, - ) == [ - (b"x-other", b"keep"), - (b"X-PolicyEngine-Request-Id", b"new"), - (b"traceparent", b"parent"), - ] diff --git a/tests/test_flask_adapter.py b/tests/test_flask_adapter.py deleted file mode 100644 index 7397746..0000000 --- a/tests/test_flask_adapter.py +++ /dev/null @@ -1,171 +0,0 @@ -from __future__ import annotations - -import pytest - -from policyengine_observability import ( - ObservabilityConfig, - ObservabilityRuntime, -) -from policyengine_observability.adapters.flask import ( - FlaskObservabilityAdapter, - _split_forwarded_for, - init_flask_observability, -) - - -class RecordingInstrument: - def __init__(self) -> None: - self.calls = [] - - def add(self, value, attributes=None) -> None: - self.calls.append(("add", value, attributes)) - - def record(self, value, attributes=None) -> None: - self.calls.append(("record", value, attributes)) - - -def _flask(): - return pytest.importorskip("flask") - - -def test_flask_adapter_records_request_metadata_and_headers() -> None: - flask = _flask() - app = flask.Flask(__name__) - runtime = ObservabilityRuntime( - ObservabilityConfig( - service_name="svc", - otel_enabled=False, - metric_attribute_keys=( - "service.name", - "route", - "method", - "status_code", - "segment", - "tool", - ), - ) - ) - runtime.requests = RecordingInstrument() - runtime.http_duration = RecordingInstrument() - runtime.active_requests = RecordingInstrument() - runtime.segment_duration = RecordingInstrument() - - @app.get("/items/") - def item(item_id: str): - runtime.set_attribute("tool", "handler") - with runtime.segment("load", tool="handler"): - pass - return {"item_id": item_id} - - initialized = init_flask_observability( - app, - runtime=runtime, - service_name="svc", - ) - - response = app.test_client().get( - "/items/abc?debug=1", - headers={ - "X-Forwarded-For": "203.0.113.1, 198.51.100.2", - "User-Agent": "pytest", - "Origin": "https://policyengine.org", - }, - ) - - assert initialized is runtime - assert response.status_code == 200 - assert response.headers["X-PolicyEngine-Request-Id"] - _, _, request_attributes = runtime.requests.calls[0] - _, _, segment_attributes = runtime.segment_duration.calls[0] - assert request_attributes["route"] == "/items/" - assert request_attributes["status_code"] == "200" - assert segment_attributes["tool"] == "handler" - assert runtime.active_requests.calls[0][1] == 1 - assert runtime.active_requests.calls[-1][1] == -1 - - -def test_flask_adapter_is_idempotent_and_honors_disabled_runtime() -> None: - flask = _flask() - app = flask.Flask(__name__) - runtime = ObservabilityRuntime.disabled() - - first = init_flask_observability( - app, - runtime=runtime, - service_name="svc", - ) - second = init_flask_observability( - app, - runtime=ObservabilityRuntime( - ObservabilityConfig(service_name="other") - ), - service_name="other", - ) - - assert first is runtime - assert second is runtime - assert "policyengine_observability_adapter" not in app.extensions - - -def test_flask_adapter_enabled_idempotence_and_start_failure() -> None: - flask = _flask() - app = flask.Flask(__name__) - runtime = ObservabilityRuntime(ObservabilityConfig(service_name="svc")) - adapter = FlaskObservabilityAdapter(runtime) - failures = [] - runtime.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - - adapter.instrument_app(app) - adapter.instrument_app(app) - adapter.start_request() - - assert app.extensions["policyengine_observability_adapter"] is adapter - assert failures == ["flask.before_request"] - - -def test_flask_init_builds_runtime_from_config_and_returns_existing() -> None: - flask = _flask() - app = flask.Flask(__name__) - - first = init_flask_observability( - app, - config=ObservabilityConfig(service_name="svc", enabled=False), - service_name="ignored", - ) - second = init_flask_observability(app, service_name="other") - - assert first is second - assert first.config.service_name == "svc" - - -def test_flask_inbound_metadata_ip_source_variants() -> None: - flask = _flask() - app = flask.Flask(__name__) - runtime = ObservabilityRuntime( - ObservabilityConfig(service_name="svc", log_raw_ip=False) - ) - adapter = FlaskObservabilityAdapter(runtime) - - with app.test_request_context( - "/", - headers={"X-Real-IP": "198.51.100.5"}, - ): - metadata = adapter._inbound_metadata(flask.request) - assert metadata["ip_source"] == "x_real_ip" - assert "client_ip" not in metadata - - with app.test_request_context( - "/", environ_base={"REMOTE_ADDR": "10.0.0.1"} - ): - metadata = adapter._inbound_metadata(flask.request) - assert metadata["ip_source"] == "remote_addr" - - -def test_split_forwarded_for_discards_empty_parts() -> None: - assert _split_forwarded_for(None) == [] - assert _split_forwarded_for(" 1.1.1.1, ,2.2.2.2 ") == [ - "1.1.1.1", - "2.2.2.2", - ] diff --git a/tests/test_google_credentials.py b/tests/test_google_credentials.py index 4870c26..7d0fb5a 100644 --- a/tests/test_google_credentials.py +++ b/tests/test_google_credentials.py @@ -1,323 +1,103 @@ from __future__ import annotations import json -import os import stat -import tempfile -import pytest +from policyengine_observability import google_credentials -from policyengine_observability.destinations import ( - google_cloud_logging, - google_credentials, -) -from policyengine_observability.destinations.google_credentials import ( - configure_google_application_credentials, - load_google_credentials, -) - -@pytest.fixture(autouse=True) -def clear_google_credential_env(monkeypatch) -> None: - for key in ( +def test_missing_and_malformed_credentials_are_nonfatal( + monkeypatch, tmp_path +) -> None: + for name in ( "GCP_CREDENTIALS_JSON", "GOOGLE_APPLICATION_CREDENTIALS", - "MODAL_IDENTITY_TOKEN", - "OBSERVABILITY_GOOGLE_OIDC_TOKEN", - "OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL", - "OBSERVABILITY_GOOGLE_STS_TOKEN_URL", - "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER", + google_credentials.MODAL_IDENTITY_TOKEN_ENV, + google_credentials.OIDC_TOKEN_ENV, + google_credentials.WORKLOAD_IDENTITY_PROVIDER_ENV, ): - monkeypatch.delenv(key, raising=False) - - -def test_configure_google_application_credentials_preserves_existing_env( - monkeypatch, -) -> None: - monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", "/existing.json") - monkeypatch.setenv("GCP_CREDENTIALS_JSON", '{"project_id":"test"}') - - path = configure_google_application_credentials() - - assert str(path) == "/existing.json" - assert os.environ["GOOGLE_APPLICATION_CREDENTIALS"] == "/existing.json" - - -def test_configure_google_application_credentials_noops_without_json( - monkeypatch, -) -> None: - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) - monkeypatch.delenv("GCP_CREDENTIALS_JSON", raising=False) - - assert configure_google_application_credentials() is None - assert "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ - - -def test_configure_google_application_credentials_materializes_json( - monkeypatch, - tmp_path, -) -> None: - credentials_path = tmp_path / "credentials.json" - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) - monkeypatch.setenv("GCP_CREDENTIALS_JSON", '{"project_id":"test"}') - - path = configure_google_application_credentials( - credentials_path=credentials_path, - ) - - assert path == credentials_path - assert credentials_path.read_text() == '{"project_id":"test"}' - assert stat.S_IMODE(credentials_path.stat().st_mode) == 0o600 - assert os.environ["GOOGLE_APPLICATION_CREDENTIALS"] == str( - credentials_path - ) - - -def test_configure_google_application_credentials_rejects_invalid_json( - monkeypatch, - tmp_path, -) -> None: - credentials_path = tmp_path / "credentials.json" - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) + monkeypatch.delenv(name, raising=False) + assert google_credentials.load_google_credentials() is None monkeypatch.setenv("GCP_CREDENTIALS_JSON", "not-json") - - path = configure_google_application_credentials( - credentials_path=credentials_path, + assert ( + google_credentials.load_google_credentials( + credentials_path=tmp_path / "credentials.json" + ) + is None ) - assert path is None - assert not credentials_path.exists() - assert "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ - -def test_configure_google_application_credentials_fails_open_on_unexpected_error( - monkeypatch, - tmp_path, -) -> None: - credentials_path = tmp_path / "credentials.json" - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) - monkeypatch.setenv("GCP_CREDENTIALS_JSON", '{"project_id":"test"}') - monkeypatch.setattr( - "policyengine_observability.destinations.google_credentials.json.loads", - lambda _value: (_ for _ in ()).throw(RuntimeError("boom")), - ) - - path = configure_google_application_credentials( - credentials_path=credentials_path, - ) - - assert path is None - assert not credentials_path.exists() - assert "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ - - -def test_configure_google_application_credentials_fails_open_on_write_error( - monkeypatch, - tmp_path, -) -> None: - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) - monkeypatch.setenv("GCP_CREDENTIALS_JSON", '{"project_id":"test"}') - - path = configure_google_application_credentials( - credentials_path=tmp_path / "missing" / "credentials.json", - ) - - assert path is None - assert "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ - - -def test_configure_google_application_credentials_materializes_oidc_wif( - monkeypatch, - tmp_path, +def test_json_credentials_are_materialized_with_private_permissions( + monkeypatch, tmp_path ) -> None: - monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path)) - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) - monkeypatch.delenv("GCP_CREDENTIALS_JSON", raising=False) - monkeypatch.setenv("OBSERVABILITY_GOOGLE_OIDC_TOKEN", "jwt-token") + path = tmp_path / "credentials.json" monkeypatch.setenv( - "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER", - "projects/123/locations/global/workloadIdentityPools/modal/providers/modal", + "GCP_CREDENTIALS_JSON", + json.dumps({"type": "external_account", "audience": "test"}), ) - monkeypatch.setenv( - "OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL", - "observability-writer@example.iam.gserviceaccount.com", + monkeypatch.setattr( + google_credentials, + "_load_credentials_from_file", + lambda loaded_path: loaded_path, ) - - path = configure_google_application_credentials() - - assert path == tmp_path / "policyengine-observability-wif.json" - assert os.environ["GOOGLE_APPLICATION_CREDENTIALS"] == str(path) + result = google_credentials.load_google_credentials(credentials_path=path) + assert result == path assert stat.S_IMODE(path.stat().st_mode) == 0o600 - token_path = tmp_path / "policyengine-observability-oidc.jwt" - assert token_path.read_text() == "jwt-token" - assert stat.S_IMODE(token_path.stat().st_mode) == 0o600 - config = json.loads(path.read_text()) - assert config == { - "type": "external_account", - "audience": ( - "//iam.googleapis.com/projects/123/locations/global/" - "workloadIdentityPools/modal/providers/modal" - ), - "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", - "token_url": "https://sts.googleapis.com/v1/token", - "credential_source": { - "file": str(token_path), - "format": {"type": "text"}, - }, - "service_account_impersonation_url": ( - "https://iamcredentials.googleapis.com/v1/projects/-/" - "serviceAccounts/observability-writer@example.iam.gserviceaccount.com" - ":generateAccessToken" - ), - } -def test_configure_google_application_credentials_preserves_full_wif_audience( - monkeypatch, - tmp_path, -) -> None: - monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path)) - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) - monkeypatch.delenv("GCP_CREDENTIALS_JSON", raising=False) - monkeypatch.setenv("OBSERVABILITY_GOOGLE_OIDC_TOKEN", "jwt-token") - monkeypatch.setenv( - "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER", - "//iam.googleapis.com/projects/123/locations/global/" - "workloadIdentityPools/modal/providers/modal", +def test_modal_workload_identity_configuration(monkeypatch, tmp_path) -> None: + monkeypatch.setattr( + google_credentials.tempfile, "gettempdir", lambda: str(tmp_path) ) - - path = configure_google_application_credentials() - - config = json.loads(path.read_text()) - assert config["audience"] == ( - "//iam.googleapis.com/projects/123/locations/global/" - "workloadIdentityPools/modal/providers/modal" + monkeypatch.setenv( + google_credentials.MODAL_IDENTITY_TOKEN_ENV, "signed-test-token" ) - assert "service_account_impersonation_url" not in config - - -def test_configure_google_application_credentials_uses_modal_identity_token( - monkeypatch, - tmp_path, -) -> None: - monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path)) - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) - monkeypatch.delenv("GCP_CREDENTIALS_JSON", raising=False) - monkeypatch.delenv("OBSERVABILITY_GOOGLE_OIDC_TOKEN", raising=False) - monkeypatch.setenv("MODAL_IDENTITY_TOKEN", "modal-jwt-token") monkeypatch.setenv( - "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER", - "projects/123/locations/global/workloadIdentityPools/modal/providers/modal", + google_credentials.WORKLOAD_IDENTITY_PROVIDER_ENV, + "projects/123/locations/global/workloadIdentityPools/modal/providers/api-v1", ) - - path = configure_google_application_credentials() - - assert path == tmp_path / "policyengine-observability-wif.json" - token_path = tmp_path / "policyengine-observability-oidc.jwt" - assert token_path.read_text() == "modal-jwt-token" - - -def test_load_google_credentials_prefers_wif_without_mutating_adc( - monkeypatch, - tmp_path, -) -> None: - monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path)) - monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", "/analytics.json") - monkeypatch.setenv("GCP_CREDENTIALS_JSON", '{"project_id":"analytics"}') - monkeypatch.setenv("MODAL_IDENTITY_TOKEN", "modal-jwt-token") monkeypatch.setenv( - "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER", - "projects/123/locations/global/workloadIdentityPools/modal/providers/modal", + google_credentials.SERVICE_ACCOUNT_EMAIL_ENV, + "modal-api-v1@central.iam.gserviceaccount.com", ) - calls = [] monkeypatch.setattr( google_credentials, "_load_credentials_from_file", - lambda path: calls.append(path) or "wif-credentials", - ) - - credentials = load_google_credentials(prefer_workload_identity=True) - - assert credentials == "wif-credentials" - assert calls == [tmp_path / "policyengine-observability-wif.json"] - assert os.environ["GOOGLE_APPLICATION_CREDENTIALS"] == "/analytics.json" - - -def test_load_google_credentials_loads_identity_pool_credentials( - monkeypatch, - tmp_path, -) -> None: - from google.auth import identity_pool - - monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path)) - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) - monkeypatch.delenv("GCP_CREDENTIALS_JSON", raising=False) - monkeypatch.setenv("MODAL_IDENTITY_TOKEN", "modal-jwt-token") - monkeypatch.setenv( - "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER", - "projects/123/locations/global/workloadIdentityPools/modal/providers/modal", + lambda path: json.loads(path.read_text()), ) - - credentials = load_google_credentials(prefer_workload_identity=True) - - assert isinstance(credentials, identity_pool.Credentials) - assert "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ - - -def test_google_destination_bootstraps_application_credentials( - monkeypatch, -) -> None: - calls = [] - monkeypatch.setattr( - google_cloud_logging, - "configure_google_application_credentials", - lambda: calls.append("configured"), - ) - monkeypatch.setattr( - google_cloud_logging, - "load_google_credentials", - lambda *, prefer_workload_identity: None, + config = google_credentials.load_google_credentials( + prefer_workload_identity=True ) - - class FakeClient: - project = "test-project" - - def logger(self, log_name): - return log_name - - destination = google_cloud_logging.GoogleCloudLoggingDestination( - project=None, - log_name="policyengine-observability", - client_factory=lambda _project, _credentials: FakeClient(), + assert config["audience"].startswith("//iam.googleapis.com/projects/123") + assert ( + "modal-api-v1@central.iam.gserviceaccount.com" + in config["service_account_impersonation_url"] ) - - assert calls == ["configured"] - assert destination.project == "test-project" - assert destination.logger == "policyengine-observability" + token_path = tmp_path / "policyengine-observability-oidc.jwt" + assert token_path.read_text() == "signed-test-token" + assert stat.S_IMODE(token_path.stat().st_mode) == 0o600 -def test_google_destination_passes_loaded_credentials(monkeypatch) -> None: +def test_existing_credentials_path_is_loaded(monkeypatch, tmp_path) -> None: + path = tmp_path / "adc.json" + path.write_text("{}") + monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", str(path)) + marker = object() monkeypatch.setattr( - google_cloud_logging, - "load_google_credentials", - lambda *, prefer_workload_identity: "wif-credentials", + google_credentials, + "_load_credentials_from_file", + lambda loaded: marker if loaded == path else None, ) + assert google_credentials.load_google_credentials() is marker - calls = [] - - class FakeClient: - project = "test-project" - def logger(self, log_name): - return log_name - - destination = google_cloud_logging.GoogleCloudLoggingDestination( - project="central-project", - log_name="policyengine-observability", - client_factory=lambda project, credentials: ( - calls.append((project, credentials)) or FakeClient() - ), +def test_workload_identity_audience_preserves_supported_forms() -> None: + full = "//iam.googleapis.com/projects/123/providers/test" + assert google_credentials._workload_identity_audience(full) == full + assert ( + google_credentials._workload_identity_audience( + "projects/123/providers/test" + ) + == "//iam.googleapis.com/projects/123/providers/test" ) - - assert calls == [("central-project", "wif-credentials")] - assert destination.logger == "policyengine-observability" + assert google_credentials._workload_identity_audience("custom") == "custom" diff --git a/tests/test_google_destination.py b/tests/test_google_destination.py deleted file mode 100644 index fffb9c0..0000000 --- a/tests/test_google_destination.py +++ /dev/null @@ -1,334 +0,0 @@ -from __future__ import annotations - -import json -import math - -import pytest - -from policyengine_observability.config import ObservabilityConfig -from policyengine_observability.destinations import ( - GoogleCloudLoggingDestination, - google_cloud_logging, - normalize_payload, -) -from policyengine_observability.destinations.base import ( - accepts_keyword, - clamped, -) - - -class Unprintable: - def __str__(self) -> str: - raise RuntimeError("cannot stringify") - - -class FakeLogger: - def __init__(self) -> None: - self.calls = [] - - def log_struct(self, payload, **kwargs) -> None: - self.calls.append((payload, kwargs)) - - -class FakeGapicApi: - def __init__(self) -> None: - self.calls = [] - - def write_log_entries(self, *args, **kwargs) -> None: - self.calls.append((args, kwargs)) - - -class FakeLoggingApi: - def __init__(self) -> None: - self._gapic_api = FakeGapicApi() - - -class FakeClient: - def __init__(self, *, gapic: bool = False) -> None: - self.project = "resolved-project" - self.fake_logger = FakeLogger() - self.log_names = [] - if gapic: - self.logging_api = FakeLoggingApi() - - def logger(self, log_name: str) -> FakeLogger: - self.log_names.append(log_name) - return self.fake_logger - - -def _google_destination(monkeypatch, client, **kwargs): - monkeypatch.setattr( - google_cloud_logging, - "load_google_credentials", - lambda *, prefer_workload_identity: None, - ) - monkeypatch.setattr( - google_cloud_logging, - "configure_google_application_credentials", - lambda: None, - ) - return GoogleCloudLoggingDestination( - project=None, - log_name="policyengine-observability", - client_factory=lambda _project, _credentials: client, - **kwargs, - ) - - -@pytest.mark.parametrize( - ("value", "expected"), - [ - (5.0, 5.0), - ("5", 5.0), - (0, 0.5), - (-3, 0.5), - (1000, 60.0), - (float("inf"), 10.0), - (float("nan"), 10.0), - (None, 10.0), - ("garbage", 10.0), - ], -) -def test_clamped_bounds_and_rejects_non_finite(value, expected) -> None: - result = clamped(value, low=0.5, high=60.0, default=10.0) - - assert result == expected - assert math.isfinite(result) - - -def test_accepts_keyword_covers_named_var_keyword_and_uninspectable() -> None: - def named(payload, *, timestamp=None): - pass - - def var_keyword(payload, **kwargs): - pass - - def blind(payload): - pass - - assert accepts_keyword(named, "timestamp") is True - assert accepts_keyword(var_keyword, "timestamp") is True - assert accepts_keyword(blind, "timestamp") is False - # Builtins without introspectable signatures degrade to False - # instead of raising at construction time. - assert accepts_keyword(min, "timestamp") is False - - -def test_normalize_payload_recursively_stringifies_unsafe_values() -> None: - normalized = normalize_payload( - { - "keep": "value", - "drop_none": None, - "bytes": b"value", - "list": [1, Unprintable()], - "nested": {"object": object()}, - } - ) - - assert normalized["keep"] == "value" - assert normalized["drop_none"] is None - assert normalized["bytes"] == "value" - assert normalized["list"] == [1, ""] - assert normalized["nested"]["object"].startswith(" None: - monkeypatch.setattr( - google_cloud_logging, - "load_google_credentials", - lambda *, prefer_workload_identity: None, - ) - monkeypatch.setattr( - google_cloud_logging, - "configure_google_application_credentials", - lambda: None, - ) - client = FakeClient() - destination = GoogleCloudLoggingDestination( - project=None, - log_name="policyengine-observability", - client_factory=lambda _project, _credentials: client, - ) - - destination.emit( - { - "schema_version": "policyengine.observability.request.v1", - "service_name": "svc", - "service_role": "api", - "environment": "production", - "request_id": "request-1", - "trace_id": "abc123", - "span_id": "def456", - "path": "/calculate", - "object": object(), - }, - log_type="request", - severity="ERROR", - ) - - payload, kwargs = client.fake_logger.calls[0] - assert client.log_names == ["policyengine-observability"] - assert payload["object"].startswith(" None: - client = FakeClient(gapic=True) - - destination = _google_destination( - monkeypatch, client, write_timeout_seconds=5.0 - ) - # The write path under log_struct funnels through this method; the - # rebinding must inject the bounded retry and per-call timeout. - client.logging_api._gapic_api.write_log_entries(request="sentinel") - - assert destination.write_timeout_seconds == 5.0 - ((args, kwargs),) = client.logging_api._gapic_api.calls - assert kwargs["request"] == "sentinel" - assert kwargs["timeout"] == 5.0 - assert kwargs["retry"].timeout == 5.0 - - -def test_google_destination_clamps_write_timeout(monkeypatch) -> None: - destination = _google_destination( - monkeypatch, FakeClient(gapic=True), write_timeout_seconds=0.0 - ) - - assert destination.write_timeout_seconds == 0.5 - - -def test_google_destination_without_gapic_transport_still_works( - monkeypatch, -) -> None: - client = FakeClient() - - destination = _google_destination(monkeypatch, client) - destination.emit({"event": "x"}, log_type="event", severity="INFO") - - assert len(client.fake_logger.calls) == 1 - - -def test_google_destination_forwards_enqueue_timestamp(monkeypatch) -> None: - from datetime import UTC, datetime - - client = FakeClient() - destination = _google_destination(monkeypatch, client) - stamp = datetime(2026, 7, 8, 12, 0, 0, tzinfo=UTC) - - destination.emit( - {"event": "x"}, log_type="event", severity="INFO", timestamp=stamp - ) - destination.emit({"event": "y"}, log_type="event", severity="INFO") - - (_, stamped_kwargs), (_, plain_kwargs) = client.fake_logger.calls - assert stamped_kwargs["timestamp"] is stamp - assert "timestamp" not in plain_kwargs - - -def test_google_destination_close_closes_client(monkeypatch) -> None: - class ClosableFakeClient(FakeClient): - def __init__(self) -> None: - super().__init__() - self.closed = 0 - - def close(self) -> None: - self.closed += 1 - - client = ClosableFakeClient() - destination = _google_destination(monkeypatch, client) - - destination.close() - - assert client.closed == 1 - - -def test_google_destination_close_tolerates_closeless_client( - monkeypatch, -) -> None: - destination = _google_destination(monkeypatch, FakeClient()) - - destination.close() # FakeClient has no close; must be a no-op - - -def test_google_destination_suppresses_instrumentation_entry( - monkeypatch, -) -> None: - logging_v2 = pytest.importorskip("google.cloud.logging_v2") - monkeypatch.setattr( - logging_v2, "_instrumentation_emitted", False, raising=False - ) - - _google_destination(monkeypatch, FakeClient()) - - assert logging_v2._instrumentation_emitted is True - - -def test_google_factory_reads_write_timeout_env(monkeypatch) -> None: - captured = {} - - class StubDestination: - def __init__(self, **kwargs) -> None: - captured.update(kwargs) - - monkeypatch.setattr( - google_cloud_logging, "GoogleCloudLoggingDestination", StubDestination - ) - monkeypatch.setenv("OBSERVABILITY_GOOGLE_WRITE_TIMEOUT_SECONDS", "2.5") - - from policyengine_observability.destinations.registry import ( - destination_strategy, - ) - - destination_strategy("google_cloud_logging").factory( - config=ObservabilityConfig(google_cloud_project="proj"), - loggers={}, - serializer=json.dumps, - ) - - assert captured["project"] == "proj" - assert captured["write_timeout_seconds"] == 2.5 - - -def test_google_factory_write_timeout_defaults_without_env( - monkeypatch, -) -> None: - captured = {} - - class StubDestination: - def __init__(self, **kwargs) -> None: - captured.update(kwargs) - - monkeypatch.setattr( - google_cloud_logging, "GoogleCloudLoggingDestination", StubDestination - ) - monkeypatch.delenv( - "OBSERVABILITY_GOOGLE_WRITE_TIMEOUT_SECONDS", raising=False - ) - - from policyengine_observability.destinations.registry import ( - destination_strategy, - ) - - destination_strategy("google_cloud_logging").factory( - config=ObservabilityConfig(google_cloud_project="proj"), - loggers={}, - serializer=json.dumps, - ) - - assert captured["write_timeout_seconds"] == 10.0 - - -# ── Stdout formatters ──────────────────────────────────────────────────── diff --git a/tests/test_log_profiles.py b/tests/test_log_profiles.py deleted file mode 100644 index ddade7a..0000000 --- a/tests/test_log_profiles.py +++ /dev/null @@ -1,292 +0,0 @@ -from __future__ import annotations - -import json - -import pytest -from fakes import RecordingLogger, make_manager - -from policyengine_observability.config import ObservabilityConfig -from policyengine_observability.destinations.queued import ( - QueuedLogDestination, -) - -PLATFORM_MARKERS = ( - "OBSERVABILITY_LOG_PROFILE", - "OBSERVABILITY_PLATFORM", - "K_SERVICE", - "MODAL_ENVIRONMENT", - "MODAL_TASK_ID", - "OBSERVABILITY_LOG_DESTINATIONS", - "OBSERVABILITY_STDOUT_FORMAT", - "OBSERVABILITY_GOOGLE_CLOUD_PROJECT", - "GOOGLE_CLOUD_PROJECT", - "GCP_PROJECT", - "GCLOUD_PROJECT", -) - - -@pytest.fixture(autouse=True) -def _clean_environment(monkeypatch): - for name in PLATFORM_MARKERS: - monkeypatch.delenv(name, raising=False) - - -def _from_env(monkeypatch, **env): - for name, value in env.items(): - monkeypatch.setenv(name, value) - return ObservabilityConfig.from_env(service_name="svc") - - -def test_explicit_gcp_agent_profile(monkeypatch) -> None: - config = _from_env(monkeypatch, OBSERVABILITY_LOG_PROFILE="gcp-agent") - - assert config.log_profile == "gcp-agent" - assert config.log_destinations == ("stdout",) - assert config.stdout_format == "google" - assert config.config_warnings == () - - -def test_profile_name_accepts_underscore_variant(monkeypatch) -> None: - config = _from_env(monkeypatch, OBSERVABILITY_LOG_PROFILE=" GCP_Agent ") - - assert config.log_profile == "gcp-agent" - assert config.config_warnings == () - - -def test_explicit_gcp_direct_profile_with_project(monkeypatch) -> None: - config = _from_env( - monkeypatch, - OBSERVABILITY_LOG_PROFILE="gcp-direct", - OBSERVABILITY_GOOGLE_CLOUD_PROJECT="proj", - ) - - assert config.log_profile == "gcp-direct" - assert config.log_destinations == ("stdout", "google_cloud_logging") - assert config.stdout_format == "plain" - assert config.config_warnings == () - - -def test_gcp_direct_without_project_downgrades_with_warning( - monkeypatch, -) -> None: - config = _from_env(monkeypatch, OBSERVABILITY_LOG_PROFILE="gcp-direct") - - assert config.log_profile == "plain-sync" - assert config.log_destinations == ("stdout",) - assert config.stdout_format == "plain" - assert len(config.config_warnings) == 1 - # The requirement comes from the strategy registration, and the - # warning names the actual profile — no hard-coded backend text. - assert "gcp-direct" in config.config_warnings[0] - assert "google_cloud_project" in config.config_warnings[0] - - -def test_explicit_plain_sync_profile_is_kill_switch(monkeypatch) -> None: - config = _from_env( - monkeypatch, - OBSERVABILITY_LOG_PROFILE="plain-sync", - OBSERVABILITY_PLATFORM="modal", - OBSERVABILITY_GOOGLE_CLOUD_PROJECT="proj", - ) - - assert config.log_profile == "plain-sync" - assert config.log_destinations == ("stdout",) - assert config.stdout_format == "plain" - - -def test_unknown_profile_falls_back_to_plain_sync_with_warning( - monkeypatch, -) -> None: - config = _from_env(monkeypatch, OBSERVABILITY_LOG_PROFILE="gcp-agnet") - - assert config.log_profile == "plain-sync" - assert config.log_destinations == ("stdout",) - assert any("gcp-agnet" in w for w in config.config_warnings) - - -def test_auto_detects_cloud_run_via_observability_platform( - monkeypatch, -) -> None: - config = _from_env(monkeypatch, OBSERVABILITY_PLATFORM="google_cloud_run") - - assert config.log_profile == "gcp-agent" - assert config.stdout_format == "google" - - -def test_auto_detects_modal_via_observability_platform(monkeypatch) -> None: - config = _from_env( - monkeypatch, - OBSERVABILITY_PLATFORM="modal", - OBSERVABILITY_GOOGLE_CLOUD_PROJECT="proj", - ) - - assert config.log_profile == "gcp-direct" - assert config.log_destinations == ("stdout", "google_cloud_logging") - - -def test_auto_detects_cloud_run_via_k_service(monkeypatch) -> None: - config = _from_env(monkeypatch, K_SERVICE="household-api") - - assert config.log_profile == "gcp-agent" - - -def test_auto_detects_modal_via_task_marker(monkeypatch) -> None: - config = _from_env( - monkeypatch, - MODAL_TASK_ID="ta-123", - OBSERVABILITY_GOOGLE_CLOUD_PROJECT="proj", - ) - - assert config.log_profile == "gcp-direct" - - -def test_auto_detects_modal_via_environment_marker(monkeypatch) -> None: - config = _from_env( - monkeypatch, - MODAL_ENVIRONMENT="main", - OBSERVABILITY_GOOGLE_CLOUD_PROJECT="proj", - ) - - assert config.log_profile == "gcp-direct" - - -def test_auto_detected_modal_without_project_downgrades(monkeypatch) -> None: - """The deployment-realistic failure: Modal markers present but no - resolvable project must land on plain-sync with a warning, exactly - like the explicit profile.""" - config = _from_env(monkeypatch, MODAL_TASK_ID="ta-123") - - assert config.log_profile == "plain-sync" - assert config.log_destinations == ("stdout",) - assert any("google_cloud_project" in w for w in config.config_warnings) - - -def test_observability_platform_beats_generic_markers(monkeypatch) -> None: - config = _from_env( - monkeypatch, - OBSERVABILITY_PLATFORM="google_cloud_run", - MODAL_TASK_ID="ta-123", - ) - - assert config.log_profile == "gcp-agent" - - -def test_auto_without_markers_preserves_caller_defaults(monkeypatch) -> None: - config = ObservabilityConfig.from_env( - service_name="svc", - default_log_destinations=("stdout", "custom"), - ) - - assert config.log_profile == "auto" - assert config.log_destinations == ("stdout", "custom") - assert config.stdout_format == "plain" - assert config.config_warnings == () - - -def test_explicit_destination_env_overrides_profile(monkeypatch) -> None: - config = _from_env( - monkeypatch, - OBSERVABILITY_LOG_PROFILE="gcp-agent", - OBSERVABILITY_LOG_DESTINATIONS="stdout,google_cloud_logging", - OBSERVABILITY_GOOGLE_CLOUD_PROJECT="proj", - ) - - assert config.log_destinations == ("stdout", "google_cloud_logging") - # The profile's other half still applies. - assert config.stdout_format == "google" - - -def test_explicit_stdout_format_env_overrides_profile(monkeypatch) -> None: - config = _from_env( - monkeypatch, - OBSERVABILITY_LOG_PROFILE="gcp-agent", - OBSERVABILITY_STDOUT_FORMAT="plain", - ) - - assert config.stdout_format == "plain" - assert config.log_destinations == ("stdout",) - - -def test_sync_profiles_build_no_queued_destinations(monkeypatch) -> None: - for profile in ("gcp-agent", "plain-sync"): - config = _from_env(monkeypatch, OBSERVABILITY_LOG_PROFILE=profile) - manager, failures = make_manager(config) - manager.configure() - - assert not any( - isinstance(destination, QueuedLogDestination) - for destination in manager.destinations - ) - assert failures == [] - - -def test_gcp_agent_profile_formats_stdout_through_manager( - monkeypatch, -) -> None: - """End-to-end wiring: the profile's formatter half must survive the - manager's build path, not just direct construction.""" - config = _from_env( - monkeypatch, - OBSERVABILITY_LOG_PROFILE="gcp-agent", - OBSERVABILITY_GOOGLE_CLOUD_PROJECT="proj", - ) - logger = RecordingLogger() - manager, failures = make_manager(config, loggers={"event": logger}) - manager.configure() - - manager.emit( - {"event": "x", "trace_id": "abc"}, log_type="event", severity="INFO" - ) - - line = json.loads(logger.lines[0][1]) - assert line["logging.googleapis.com/labels"]["log_type"] == "event" - assert line["logging.googleapis.com/trace"] == "projects/proj/traces/abc" - assert failures == [] - - -def test_manager_reports_profile_warnings_once(monkeypatch) -> None: - config = _from_env(monkeypatch, OBSERVABILITY_LOG_PROFILE="bogus") - manager, failures = make_manager(config) - manager.configure() - - warnings = [ - str(exc) - for operation, exc, _fields in failures - if operation == "logging.profile_config" - ] - assert len(warnings) == 1 - assert "bogus" in warnings[0] - manager.close() - - -def test_gcp_direct_profile_builds_queued_google(monkeypatch) -> None: - from policyengine_observability.destinations import ( - google_cloud_logging as google_module, - ) - - class StubGoogle: - name = "google_cloud_logging" - - def __init__(self, **kwargs) -> None: - self.kwargs = kwargs - - def emit(self, payload, *, log_type, severity, timestamp=None): - pass - - monkeypatch.setattr( - google_module, "GoogleCloudLoggingDestination", StubGoogle - ) - config = _from_env( - monkeypatch, - OBSERVABILITY_LOG_PROFILE="gcp-direct", - OBSERVABILITY_GOOGLE_CLOUD_PROJECT="proj", - ) - manager, failures = make_manager(config) - manager.configure() - - stdout_destination, queued = manager.destinations - assert isinstance(queued, QueuedLogDestination) - assert isinstance(queued.inner, StubGoogle) - assert queued.inner.kwargs["project"] == "proj" - assert failures == [] - manager.close() diff --git a/tests/test_otel.py b/tests/test_otel.py new file mode 100644 index 0000000..03ccefc --- /dev/null +++ b/tests/test_otel.py @@ -0,0 +1,614 @@ +from __future__ import annotations + +import asyncio +import io +from datetime import UTC, datetime, timedelta + +from conftest import make_config, records +from opentelemetry.sdk.trace.export import ( + SimpleSpanProcessor, + SpanExportResult, +) +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + +from policyengine_observability import OTelConfig, configure +from policyengine_observability.diagnostics import Diagnostics +from policyengine_observability.otel import ( + OTelRuntime, + SpanHandle, + _build_metric_exporter, + _build_span_exporter, + _exporter_kwargs, + _google_grpc_credentials, + _google_http_session, + _google_id_token_credentials, + _http_signal_endpoint, + _lazy_metric_exporter_class, + _LazySpanExporter, + _workload_identity_audience, + _write_subject_token, + captured_at_is_recent, +) + + +def _runtime_with_spans(): + config = make_config(otel=OTelConfig(enabled=True, endpoint=None)) + runtime = configure(config) + runtime._delivery._stdout = io.StringIO() + exporter = InMemorySpanExporter() + runtime._otel._tracer_provider.add_span_processor( + SimpleSpanProcessor(exporter) + ) + return runtime, exporter + + +def test_owned_provider_creates_local_spans_metrics_and_resources() -> None: + runtime, exporter = _runtime_with_spans() + resource = runtime._otel.resource_attributes() + assert resource["service.name"] == "test-api" + assert resource["deployment.environment.name"] == "test" + assert runtime._otel.tracer is not None + assert runtime._otel.meter is not None + with runtime.operation("simulation.run"): + with runtime.span("simulation.calculate"): + runtime.event("calculation.started") + runtime._otel.force_flush(1) + spans = exporter.get_finished_spans() + assert {span.name for span in spans} == { + "simulation.run", + "simulation.calculate", + } + child = next(span for span in spans if span.name == "simulation.calculate") + parent = next(span for span in spans if span.name == "simulation.run") + assert child.parent.span_id == parent.context.span_id + runtime.shutdown() + + +def test_incoming_trace_context_correlates_log_and_response() -> None: + runtime, exporter = _runtime_with_spans() + trace_id = "1" * 32 + parent_id = "2" * 16 + runtime.begin_request( + headers={"traceparent": f"00-{trace_id}-{parent_id}-01"}, + method="GET", + route="/health", + ) + headers = runtime.response_headers() + assert headers["traceparent"].startswith(f"00-{trace_id}-") + runtime.end_request(status_code=200) + item = records(runtime._delivery._stdout)[0] + assert item["trace_id"] == trace_id + assert item["logging.googleapis.com/trace"].endswith(trace_id) + span = exporter.get_finished_spans()[0] + assert span.parent.span_id == int(parent_id, 16) + runtime.shutdown() + + +def test_malformed_trace_context_starts_new_trace() -> None: + runtime, exporter = _runtime_with_spans() + runtime.begin_request( + headers={"traceparent": "malformed"}, + method="GET", + route="/health", + ) + runtime.end_request(status_code=200) + span = exporter.get_finished_spans()[0] + assert span.context.trace_id != 0 + assert span.parent is None + runtime.shutdown() + + +def test_recent_async_context_is_parent_and_old_context_is_link() -> None: + runtime, exporter = _runtime_with_spans() + trace_id = "3" * 32 + parent_id = "4" * 16 + recent = { + "traceparent": f"00-{trace_id}-{parent_id}-01", + "request_id": "dispatch-request", + "captured_at": datetime.now(UTC).isoformat(), + } + with runtime.operation("recent", remote_context=recent): + runtime.event("recent.event") + + old = { + **recent, + "captured_at": (datetime.now(UTC) - timedelta(minutes=10)).isoformat(), + } + with runtime.operation("old", remote_context=old): + pass + spans = {span.name: span for span in exporter.get_finished_spans()} + assert spans["recent"].parent.span_id == int(parent_id, 16) + assert spans["old"].parent is None + assert spans["old"].links[0].context.span_id == int(parent_id, 16) + runtime.shutdown() + + +def test_retry_forces_span_link_even_when_context_is_recent() -> None: + runtime, exporter = _runtime_with_spans() + remote = { + "traceparent": f"00-{'5' * 32}-{'6' * 16}-01", + "captured_at": datetime.now(UTC).isoformat(), + } + with runtime.operation( + "retry", remote_context=remote, independent_retry=True + ): + pass + span = exporter.get_finished_spans()[0] + assert span.parent is None + assert len(span.links) == 1 + runtime.shutdown() + + +def test_external_provider_mode_uses_caller_provider(monkeypatch) -> None: + class Provider: + def get_tracer(self, *_args): + return object() + + class MeterProvider: + def get_meter(self, *_args): + return None + + tracer_provider = Provider() + meter_provider = MeterProvider() + monkeypatch.setattr( + "opentelemetry.trace.get_tracer_provider", lambda: tracer_provider + ) + monkeypatch.setattr( + "opentelemetry.metrics.get_meter_provider", lambda: meter_provider + ) + otel = OTelRuntime( + make_config(otel=OTelConfig(enabled=True, provider_mode="external")), + Diagnostics(), + ) + assert otel._tracer_provider is tracer_provider + assert otel._meter_provider is meter_provider + assert not otel._owns_tracer_provider + + +def test_lazy_span_exporter_contains_failure() -> None: + diagnostics = Diagnostics() + exporter = _LazySpanExporter( + lambda: (_ for _ in ()).throw(ConnectionError("dns failure")), + diagnostics, + ) + assert exporter.export([]) is SpanExportResult.FAILURE + assert diagnostics.count("spans.export_failure") == 1 + assert exporter.force_flush() + exporter.shutdown() + + +def test_lazy_metric_exporter_contains_failure() -> None: + from opentelemetry.sdk.metrics.export import MetricExportResult + + diagnostics = Diagnostics() + exporter = _lazy_metric_exporter_class()( + lambda: (_ for _ in ()).throw(PermissionError("denied")), + diagnostics, + ) + assert exporter.export(object()) is MetricExportResult.FAILURE + assert diagnostics.count("metrics.export_failure") == 1 + assert exporter.force_flush() + exporter.shutdown() + + +def test_async_context_age_and_http_endpoint_helpers() -> None: + assert captured_at_is_recent(datetime.now(UTC).isoformat(), 300) + assert not captured_at_is_recent("invalid", 300) + assert not captured_at_is_recent(None, 300) + assert not captured_at_is_recent( + (datetime.now(UTC) + timedelta(seconds=10)).isoformat(), 300 + ) + assert _http_signal_endpoint("https://collector", "traces") == ( + "https://collector/v1/traces" + ) + assert ( + _http_signal_endpoint("https://collector/v1/traces", "traces") + == "https://collector/v1/traces" + ) + + +def test_otlp_exporter_builders_apply_protocol_endpoint_and_timeout( + monkeypatch, +) -> None: + from opentelemetry.exporter.otlp.proto.grpc import ( + metric_exporter as grpc_metric, + ) + from opentelemetry.exporter.otlp.proto.grpc import ( + trace_exporter as grpc_trace, + ) + from opentelemetry.exporter.otlp.proto.http import ( + metric_exporter as http_metric, + ) + from opentelemetry.exporter.otlp.proto.http import ( + trace_exporter as http_trace, + ) + + created: list[tuple[str, dict]] = [] + + def constructor(name): + return lambda **kwargs: created.append((name, kwargs)) or name + + monkeypatch.setattr( + grpc_trace, "OTLPSpanExporter", constructor("grpc-span") + ) + monkeypatch.setattr( + grpc_metric, "OTLPMetricExporter", constructor("grpc-metric") + ) + monkeypatch.setattr( + http_trace, "OTLPSpanExporter", constructor("http-span") + ) + monkeypatch.setattr( + http_metric, "OTLPMetricExporter", constructor("http-metric") + ) + grpc = OTelConfig( + endpoint="collector:4317", + protocol="grpc", + headers=(("x-test", "value"),), + export_timeout_seconds=2, + ) + assert _build_span_exporter(grpc) == "grpc-span" + assert _build_metric_exporter(grpc) == "grpc-metric" + http = OTelConfig( + endpoint="https://collector/", + protocol="http/protobuf", + export_timeout_seconds=3, + ) + assert _build_span_exporter(http) == "http-span" + assert _build_metric_exporter(http) == "http-metric" + assert created[2][1]["endpoint"] == "https://collector/v1/traces" + assert created[3][1]["endpoint"] == "https://collector/v1/metrics" + assert created[0][1]["headers"] == {"x-test": "value"} + assert created[0][1]["timeout"] == 2 + + +def test_google_exporter_kwargs_select_protocol_credentials( + monkeypatch, +) -> None: + monkeypatch.setattr( + "policyengine_observability.otel._google_grpc_credentials", + lambda audience: f"grpc:{audience}", + ) + monkeypatch.setattr( + "policyengine_observability.otel._google_http_session", + lambda audience, headers: (audience, dict(headers)), + ) + grpc = _exporter_kwargs( + OTelConfig( + endpoint="collector:4317", + google_audience="https://collector", + ) + ) + assert grpc["credentials"] == "grpc:https://collector" + http = _exporter_kwargs( + OTelConfig( + endpoint="https://collector", + protocol="http/protobuf", + google_audience="https://collector", + headers=(("x", "y"),), + ) + ) + assert http["session"] == ("https://collector", {"x": "y"}) + assert "headers" not in http + + +def test_google_id_token_uses_modal_workload_identity(monkeypatch) -> None: + from google.auth import identity_pool, impersonated_credentials + + calls: dict[str, object] = {} + + class Source: + @classmethod + def from_info(cls, config, scopes): + calls["source"] = (config, scopes) + return "source-credentials" + + class Target: + def __init__(self, **kwargs): + calls["target"] = kwargs + + class Identity: + def __init__(self, **kwargs): + calls["identity"] = kwargs + + monkeypatch.setattr(identity_pool, "Credentials", Source) + monkeypatch.setattr(impersonated_credentials, "Credentials", Target) + monkeypatch.setattr( + impersonated_credentials, "IDTokenCredentials", Identity + ) + monkeypatch.setattr( + "policyengine_observability.otel._write_subject_token", + lambda token: f"/tmp/{token}", + ) + monkeypatch.setenv("MODAL_IDENTITY_TOKEN", "modal-token") + monkeypatch.setenv( + "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER", + "projects/123/providers/api-v1", + ) + monkeypatch.setenv( + "OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL", + "modal@central.iam.gserviceaccount.com", + ) + result = _google_id_token_credentials("https://collector") + assert isinstance(result, Identity) + assert calls["source"][0]["credential_source"]["file"] == ( + "/tmp/modal-token" + ) + assert calls["identity"]["target_audience"] == "https://collector" + + +def test_google_id_token_falls_back_to_application_credentials( + monkeypatch, +) -> None: + from google.oauth2 import id_token + + monkeypatch.delenv("MODAL_IDENTITY_TOKEN", raising=False) + monkeypatch.delenv("OBSERVABILITY_GOOGLE_OIDC_TOKEN", raising=False) + monkeypatch.setattr( + id_token, + "fetch_id_token_credentials", + lambda audience, request: (audience, request), + ) + result = _google_id_token_credentials("https://collector") + assert result[0] == "https://collector" + + +def test_google_transport_helpers(monkeypatch, tmp_path) -> None: + import grpc + from google.auth.transport import grpc as google_grpc + from google.auth.transport import requests as google_requests + + monkeypatch.setattr( + "policyengine_observability.otel._google_id_token_credentials", + lambda audience: f"token:{audience}", + ) + monkeypatch.setattr( + google_grpc, + "AuthMetadataPlugin", + lambda credentials, request, default_host: ( + credentials, + default_host, + ), + ) + monkeypatch.setattr(grpc, "ssl_channel_credentials", lambda: "ssl") + monkeypatch.setattr( + grpc, "metadata_call_credentials", lambda plugin: ("metadata", plugin) + ) + monkeypatch.setattr( + grpc, + "composite_channel_credentials", + lambda ssl, metadata: (ssl, metadata), + ) + assert _google_grpc_credentials("https://collector.example") == ( + "ssl", + ( + "metadata", + ("token:https://collector.example", "collector.example"), + ), + ) + + class Session: + def __init__(self, credentials): + self.credentials = credentials + self.headers = {} + + monkeypatch.setattr(google_requests, "AuthorizedSession", Session) + session = _google_http_session("https://collector", {"x": "y"}) + assert session.credentials == "token:https://collector" + assert session.headers == {"x": "y"} + + monkeypatch.setattr("tempfile.gettempdir", lambda: str(tmp_path)) + path = _write_subject_token("short-lived-token") + assert open(path).read() == "short-lived-token" + assert _workload_identity_audience("projects/123/provider") == ( + "//iam.googleapis.com/projects/123/provider" + ) + + +def test_otel_runtime_contains_span_and_propagation_failures( + monkeypatch, +) -> None: + from opentelemetry import trace + from opentelemetry.trace.propagation.tracecontext import ( + TraceContextTextMapPropagator, + ) + + diagnostics = Diagnostics() + otel = OTelRuntime( + make_config(otel=OTelConfig(enabled=False)), diagnostics + ) + + class BrokenTracer: + def start_as_current_span(self, *_args, **_kwargs): + raise RuntimeError("start failed") + + otel.tracer = BrokenTracer() + assert otel.start_span("broken") is None + + monkeypatch.setattr( + trace, + "get_current_span", + lambda *_args: (_ for _ in ()).throw(RuntimeError("context failed")), + ) + assert otel.current_correlation() == {} + otel.set_span_attributes({"key": "value"}) + assert otel.remote_span_context({}) is None + + monkeypatch.setattr( + TraceContextTextMapPropagator, + "extract", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + ValueError("malformed") + ), + ) + monkeypatch.setattr( + TraceContextTextMapPropagator, + "inject", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + ValueError("unavailable") + ), + ) + assert otel.extract({"traceparent": "bad"}) is None + otel.inject({}) + assert diagnostics.count("failure.otel.span_start") == 1 + assert diagnostics.count("failure.otel.context_extract") == 1 + assert diagnostics.count("failure.otel.context_inject") == 1 + + +def test_otel_end_span_metrics_flush_and_shutdown_fail_open() -> None: + diagnostics = Diagnostics() + otel = OTelRuntime( + make_config(otel=OTelConfig(enabled=False)), diagnostics + ) + + class Span: + def __init__(self): + self.recorded = None + self.status = None + + def record_exception(self, error): + self.recorded = error + + def set_status(self, status): + self.status = status + + class Manager: + def __init__(self): + self.args = None + + def __exit__(self, *args): + self.args = args + + span = Span() + manager = Manager() + error = ValueError("application") + otel.end_span(SpanHandle(manager=manager, span=span), error) + assert span.recorded is error + assert manager.args[1] is error + otel.end_span(None) + + class Instrument: + def add(self, *_args, **_kwargs): + raise RuntimeError("metric failed") + + otel._error_count = Instrument() + otel.record_error({"outcome": "error"}) + otel.record_dropped("log") + + class Provider: + def force_flush(self, **_kwargs): + raise TimeoutError("flush") + + def shutdown(self, **_kwargs): + raise TimeoutError("shutdown") + + provider = Provider() + otel._tracer_provider = provider + otel._meter_provider = provider + otel._owns_tracer_provider = True + otel._owns_meter_provider = True + otel.force_flush(0.01) + otel.shutdown(0.01) + assert diagnostics.count("failure.otel.metric_record") == 1 + assert diagnostics.count("failure.otel.force_flush") == 2 + assert diagnostics.count("failure.otel.trace_shutdown") == 1 + assert diagnostics.count("failure.otel.metric_shutdown") == 1 + + +def test_lazy_exporters_delegate_flush_and_shutdown_failures() -> None: + diagnostics = Diagnostics() + + class SpanDelegate: + def export(self, spans): + return SpanExportResult.SUCCESS + + def force_flush(self, _timeout): + raise TimeoutError("flush") + + def shutdown(self): + raise RuntimeError("shutdown") + + span_exporter = _LazySpanExporter(lambda: SpanDelegate(), diagnostics) + assert span_exporter.export([]) is SpanExportResult.SUCCESS + assert not span_exporter.force_flush() + span_exporter.shutdown() + + class MetricDelegate: + def export(self, *_args, **_kwargs): + from opentelemetry.sdk.metrics.export import MetricExportResult + + return MetricExportResult.SUCCESS + + def force_flush(self, _timeout): + raise TimeoutError("flush") + + def shutdown(self, **_kwargs): + raise RuntimeError("shutdown") + + metric_exporter = _lazy_metric_exporter_class()( + lambda: MetricDelegate(), diagnostics + ) + from opentelemetry.sdk.metrics.export import MetricExportResult + + assert metric_exporter.export(object()) is MetricExportResult.SUCCESS + assert not metric_exporter.force_flush() + metric_exporter.shutdown() + assert diagnostics.count("failure.otel.span_flush") == 1 + assert diagnostics.count("failure.otel.span_exporter_shutdown") == 1 + assert diagnostics.count("failure.otel.metric_flush") == 1 + assert diagnostics.count("failure.otel.metric_exporter_shutdown") == 1 + + +def test_queue_depth_observation_is_bounded_and_failure_is_local() -> None: + diagnostics = Diagnostics() + otel = OTelRuntime( + make_config(otel=OTelConfig(enabled=False)), + diagnostics, + queue_depth=lambda: -4, + ) + observation = otel._observe_queue_depth(None)[0] + assert observation.value == 0 + otel._queue_depth_callback = lambda: (_ for _ in ()).throw( + RuntimeError("queue unavailable") + ) + assert otel._observe_queue_depth(None) == [] + assert diagnostics.count("failure.otel.queue_depth") == 1 + + +def test_concurrent_requests_keep_trace_and_span_context_separate() -> None: + runtime, exporter = _runtime_with_spans() + + async def request(request_id: str) -> None: + runtime.begin_request( + headers={"X-PolicyEngine-Request-Id": request_id}, + method="GET", + route="/concurrent", + ) + async with runtime.span(f"child.{request_id}"): + await asyncio.sleep(0) + runtime.event("inside") + runtime.end_request(status_code=200) + + async def run() -> None: + await asyncio.gather(request("one"), request("two")) + + asyncio.run(run()) + completion_records = [ + item + for item in records(runtime._delivery._stdout) + if item.get("event.name") == "request.completed" + ] + traces = { + item["request.id"]: item["trace_id"] for item in completion_records + } + assert traces["one"] != traces["two"] + child_spans = { + span.name: span + for span in exporter.get_finished_spans() + if span.parent + } + for request_id in ("one", "two"): + child = child_spans[f"child.{request_id}"] + assert format(child.context.trace_id, "032x") == traces[request_id] + runtime.shutdown() diff --git a/tests/test_public_api.py b/tests/test_public_api.py deleted file mode 100644 index a24f474..0000000 --- a/tests/test_public_api.py +++ /dev/null @@ -1,126 +0,0 @@ -from __future__ import annotations - -import asyncio -from enum import StrEnum - -import policyengine_observability as observability -from policyengine_observability import ( - ObservabilityConfig, - ObservabilityRuntime, -) - - -class SegmentName(StrEnum): - LOAD = "load" - - -class RecordingInstrument: - def __init__(self) -> None: - self.calls = [] - - def add(self, value, attributes=None) -> None: - self.calls.append(("add", value, attributes)) - - def record(self, value, attributes=None) -> None: - self.calls.append(("record", value, attributes)) - - -def test_public_wrappers_delegate_to_configured_runtime() -> None: - runtime = ObservabilityRuntime( - ObservabilityConfig(service_name="svc"), - segment_registry=SegmentName, - ) - runtime.operation_duration = RecordingInstrument() - runtime.segment_duration = RecordingInstrument() - runtime.operations = RecordingInstrument() - runtime.errors = RecordingInstrument() - runtime.failover_events = RecordingInstrument() - observability.set_observability_runtime(runtime) - - with observability.operation("job", flavor="cli"): - observability.set_attribute("tool", "worker") - observability.record_event("fallback_selected", reason="forced") - observability.record_error( - RuntimeError("handled"), - handled=True, - include_stack=False, - ) - with observability.segment(SegmentName.LOAD, tool="loader"): - observability.mark("custom_ms", 12.3) - observability.mark_ttft() - observability.mark_ttft_attribute() - handle = observability.start_scope({}, name="nested") - observability.annotate(handle, model="claude") - observability.end_scope(handle) - - assert observability.current_context() is None - assert observability.current_operation() is None - assert observability.traceparent_header() is None - assert observability.capture_context() is None - assert runtime.operation_duration.calls - assert runtime.segment_duration.calls - assert runtime.errors.calls - assert runtime.failover_events.calls - - -def test_public_decorators_support_sync_and_async_functions() -> None: - runtime = ObservabilityRuntime(ObservabilityConfig(service_name="svc")) - runtime.operation_duration = RecordingInstrument() - runtime.segment_duration = RecordingInstrument() - runtime.operations = RecordingInstrument() - observability.set_observability_runtime(runtime) - - @observability.entrypoint("sync_job", flavor="cli") - def sync_job() -> str: - return "sync" - - @observability.segment("async_step", flavor="worker") - async def async_step() -> str: - return "async" - - assert sync_job() == "sync" - assert asyncio.run(async_step()) == "async" - observability.instrument_fastapi(object()) - observability.instrument_httpx() - observability.shutdown_tracing() - observability.shutdown_observability() - observability.restart_observability() - - assert runtime.operation_duration.calls - assert runtime.segment_duration.calls - assert runtime.log_destination_manager.configured is True - - -def test_setting_public_runtime_clears_stale_context() -> None: - old_runtime = ObservabilityRuntime(ObservabilityConfig(service_name="old")) - observability.set_observability_runtime(old_runtime) - handle = old_runtime.start_operation("old_job") - - try: - new_runtime = ObservabilityRuntime( - ObservabilityConfig(service_name="new") - ) - observability.set_observability_runtime(new_runtime) - - assert observability.current_operation() is None - assert observability.current_context() is None - finally: - old_runtime.end_operation(handle) - - -def test_public_async_segment_and_collect_timings_wrappers() -> None: - runtime = ObservabilityRuntime(ObservabilityConfig(service_name="svc")) - observability.set_observability_runtime(runtime) - - async def run() -> dict[str, float]: - with observability.collect_timings("job") as timings: - async with observability.asegment("load"): - pass - return timings - - assert "load_ms" in asyncio.run(run()) - - -def test_registration_hooks_are_exported_at_top_level() -> None: - assert callable(observability.register_destination) - assert callable(observability.register_stdout_formatter) diff --git a/tests/test_queued_destination.py b/tests/test_queued_destination.py deleted file mode 100644 index b5b5364..0000000 --- a/tests/test_queued_destination.py +++ /dev/null @@ -1,472 +0,0 @@ -"""QueuedLogDestination tests. - -Determinism strategy: real listener threads are used, but sequencing is -via threading.Event only — close()/stop() drain to a sentinel enqueued -behind the records, so "close then assert delivered" is deterministic. -The only real-time waits are bounded joins that are themselves the -behavior under test, with generous ceilings. -""" - -from __future__ import annotations - -import time -from datetime import UTC, datetime - -import pytest -from fakes import ( - BlockingDestination, - FailingDestination, - RecordingDestination, - TimestampBlindDestination, - make_manager, -) - -from policyengine_observability.config import ObservabilityConfig -from policyengine_observability.destinations.queued import ( - QueuedLogDestination, -) -from policyengine_observability.destinations.registry import ( - destination_strategy, -) - - -def _queued(inner, **kwargs): - failures = [] - destination = QueuedLogDestination( - inner=inner, - on_failure=lambda operation, exc, **fields: failures.append( - (operation, fields) - ), - **kwargs, - ) - return destination, failures - - -def _raise_on_failure(*args, **kwargs): - raise RuntimeError("reporting channel is broken") - - -def _release_abandoned_listener(destination, blocking_inner) -> None: - """Let an abandoned listener thread exit so tests do not leak it.""" - blocking_inner.release.set() - try: - destination._queue.put_nowait(None) # the stdlib sentinel - except Exception: - pass - - -def test_close_drains_all_records_in_order_with_enqueue_timestamps() -> None: - inner = RecordingDestination() - destination, failures = _queued(inner) - before = datetime.now(UTC) - - for index in range(5): - destination.emit({"index": index}, log_type="event", severity="INFO") - destination.close(5.0) - - assert [payload["index"] for payload, *_ in inner.calls] == [ - 0, - 1, - 2, - 3, - 4, - ] - for _payload, log_type, severity, timestamp in inner.calls: - assert log_type == "event" - assert severity == "INFO" - assert timestamp.tzinfo is not None - assert before <= timestamp <= datetime.now(UTC) - assert failures == [] - - -def test_timestamp_not_forwarded_to_blind_inner() -> None: - inner = TimestampBlindDestination() - destination, failures = _queued(inner) - - destination.emit({"event": "x"}, log_type="event", severity="INFO") - destination.close(5.0) - - assert inner.calls == [({"event": "x"}, "event", "INFO")] - assert failures == [] - - -def test_timestamp_forwarded_to_var_keyword_inner() -> None: - class KwargsDestination: - name = "kwargs" - - def __init__(self) -> None: - self.kwargs = [] - - def emit(self, payload, **kwargs) -> None: - self.kwargs.append(kwargs) - - inner = KwargsDestination() - destination, failures = _queued(inner) - - destination.emit({"event": "x"}, log_type="event", severity="INFO") - destination.close(5.0) - - assert inner.kwargs[0]["timestamp"].tzinfo is not None - assert failures == [] - - -def test_uninspectable_inner_emit_degrades_to_timestamp_blind() -> None: - class BuiltinEmitDestination: - # inspect.signature(min) raises; construction must survive and - # fall back to timestamp-blind delivery. - name = "builtin-emit" - emit = min - - destination, _failures = _queued(BuiltinEmitDestination()) - - handler = destination._listener.handlers[0] - assert handler.forward_timestamp is False - destination.close(5.0) - - -def test_payload_snapshot_taken_at_enqueue() -> None: - inner = RecordingDestination() - destination, _failures = _queued(inner) - payload = {"nested": {"value": 1}} - - destination.emit(payload, log_type="event", severity="INFO") - payload["nested"]["value"] = 2 - destination.close(5.0) - - ((delivered, *_),) = inner.calls - assert delivered["nested"]["value"] == 1 - - -def test_overflow_drops_newest_and_throttles_reports() -> None: - inner = BlockingDestination() - destination, failures = _queued(inner, maxsize=10, drop_report_interval=3) - - destination.emit({"index": 0}, log_type="event", severity="INFO") - assert inner.entered.wait(5) - # The listener holds record 0; fill the 10-slot queue, then drop 4. - started = time.monotonic() - for index in range(1, 15): - destination.emit({"index": index}, log_type="event", severity="INFO") - assert time.monotonic() - started < 1.0 - - drops = [fields for op, fields in failures if op == "logging.queue_drop"] - assert [d["dropped_total"] for d in drops] == [1, 3] - assert all(d["reason"] == "full" for d in drops) - assert all(d["queue_maxsize"] == 10 for d in drops) - assert destination._drops.count == 4 - - inner.release.set() - destination.close(5.0) - - -def test_close_with_stuck_write_is_bounded_and_terminal() -> None: - inner = BlockingDestination() - destination, failures = _queued(inner) - destination.emit({"index": 0}, log_type="event", severity="INFO") - assert inner.entered.wait(5) - destination.emit({"index": 1}, log_type="event", severity="INFO") - - started = time.monotonic() - destination.close(0.05) - elapsed = time.monotonic() - started - - assert elapsed < 1.0 - timeouts = [ - fields - for op, fields in failures - if op == "logging.queue_close_timeout" - ] - assert len(timeouts) == 1 - assert timeouts[0]["pending_records"] >= 1 - - destination.emit({"index": 2}, log_type="event", severity="INFO") - drops = [fields for op, fields in failures if op == "logging.queue_drop"] - assert drops[-1]["reason"] == "closed" - - _release_abandoned_listener(destination, inner) - - -def test_timed_out_close_leaves_inner_unclosed() -> None: - """An abandoned listener may be mid-write; the inner destination - must not be closed underneath it.""" - inner = BlockingDestination() - destination, _failures = _queued(inner) - destination.emit({"index": 0}, log_type="event", severity="INFO") - assert inner.entered.wait(5) - - destination.close(0.05) - - assert inner.closed == 0 - _release_abandoned_listener(destination, inner) - - -def test_close_with_sentinel_blocked_by_full_queue_is_bounded() -> None: - inner = BlockingDestination() - destination, failures = _queued(inner, maxsize=10) - destination.emit({"index": 0}, log_type="event", severity="INFO") - assert inner.entered.wait(5) - for index in range(1, 11): - destination.emit({"index": index}, log_type="event", severity="INFO") - - started = time.monotonic() - destination.close(0.05) - elapsed = time.monotonic() - started - - assert elapsed < 1.0 - assert any(op == "logging.queue_close_timeout" for op, _fields in failures) - - _release_abandoned_listener(destination, inner) - - -def test_listener_survives_write_failures_and_throttles_reports() -> None: - inner = FailingDestination(fail_first=3) - destination, failures = _queued(inner) - - for index in range(5): - destination.emit({"index": index}, log_type="event", severity="INFO") - destination.close(5.0) - - assert [p["index"] for p in inner.delivered] == [3, 4] - writes = [fields for op, fields in failures if op == "logging.queue_write"] - # Throttle: failure 1 reports, failures 2-3 are under the interval. - assert [w["write_failures_total"] for w in writes] == [1] - assert writes[0]["destination"] == "failing" - - -def test_listener_survives_broken_reporting_channel() -> None: - """The write-failure report itself is guarded on the listener - thread: a raising on_failure must not kill delivery.""" - inner = FailingDestination(fail_first=1) - destination = QueuedLogDestination( - inner=inner, on_failure=_raise_on_failure - ) - - destination.emit({"index": 0}, log_type="event", severity="INFO") - destination.emit({"index": 1}, log_type="event", severity="INFO") - destination.close(5.0) - - assert [p["index"] for p in inner.delivered] == [1] - - -def test_emit_never_raises() -> None: - inner = RecordingDestination() - destination = QueuedLogDestination( - inner=inner, on_failure=_raise_on_failure - ) - - poisoned = {} - poisoned["self"] = poisoned # RecursionError inside normalize - destination.emit(poisoned, log_type="event", severity="INFO") - - destination.close(5.0) - destination.emit({"event": "x"}, log_type="event", severity="INFO") - - blocked = BlockingDestination() - full_destination = QueuedLogDestination( - inner=blocked, on_failure=_raise_on_failure, maxsize=10 - ) - for index in range(12): - full_destination.emit( - {"index": index}, log_type="event", severity="INFO" - ) - - blocked.release.set() - full_destination.close(5.0) - - -def test_atexit_registered_on_construction_unregistered_on_close( - monkeypatch, -) -> None: - registered = [] - unregistered = [] - monkeypatch.setattr( - "policyengine_observability.destinations.queued.atexit.register", - lambda func: registered.append(func), - ) - monkeypatch.setattr( - "policyengine_observability.destinations.queued.atexit.unregister", - lambda func: unregistered.append(func), - ) - inner = RecordingDestination() - - destination, failures = _queued(inner) - destination.close(5.0) - destination.close(5.0) # idempotent: no second stop, no reports - - assert registered == [destination.close] - assert unregistered == [destination.close] - assert failures == [] - - -def test_close_forwards_to_inner_close_only_when_drained() -> None: - class ClosableInner(RecordingDestination): - def __init__(self) -> None: - super().__init__() - self.closed = 0 - - def close(self) -> None: - self.closed += 1 - - inner = ClosableInner() - destination, _failures = _queued(inner) - destination.emit({"event": "x"}, log_type="event", severity="INFO") - destination.close(5.0) - - assert inner.closed == 1 - - -def test_inner_close_failure_is_reported_not_raised() -> None: - class ExplodingClose(RecordingDestination): - def close(self) -> None: - raise RuntimeError("client teardown failed") - - destination, failures = _queued(ExplodingClose()) - - destination.close(5.0) # must not raise (atexit calls this) - - assert any(op == "logging.destination_close" for op, _ in failures) - - -def test_knobs_are_clamped() -> None: - inner = RecordingDestination() - destination, _failures = _queued( - inner, maxsize=float("inf"), close_timeout_seconds=-5 - ) - - assert destination.maxsize == 1000 - assert destination.close_timeout_seconds == 0.0 - destination.close(5.0) - - -@pytest.mark.parametrize( - ("kwargs", "attribute", "expected"), - [ - ({"maxsize": 3}, "maxsize", 10), - ({"maxsize": 10**6}, "maxsize", 100_000), - ({"close_timeout_seconds": 100}, "close_timeout_seconds", 30.0), - ], -) -def test_knob_boundaries_are_pinned(kwargs, attribute, expected) -> None: - """Pin the clamp bounds so a low/high swap cannot pass silently.""" - destination, _failures = _queued(RecordingDestination(), **kwargs) - - assert getattr(destination, attribute) == expected - destination.close(5.0) - - -# ── Destination registry ───────────────────────────────────────────────── - - -def test_registry_builds_remote_wrapped_and_inline_bare( - fake_remote_strategy, -) -> None: - manager, _failures = make_manager( - ObservabilityConfig(log_destinations=(fake_remote_strategy, "stdout")) - ) - manager.configure() - - remote, inline = manager.destinations - assert isinstance(remote, QueuedLogDestination) - assert remote.name == "queued_recording" - assert not isinstance(inline, QueuedLogDestination) - manager.close() - - -def test_registry_unknown_name_reports_config_failure() -> None: - manager, failures = make_manager( - ObservabilityConfig(log_destinations=("nonexistent",)) - ) - manager.configure() - - assert any( - op == "logging.destination_config" - and fields.get("destination") == "nonexistent" - for op, _exc, fields in failures - ) - - -def test_google_strategy_registered_as_remote_with_aliases() -> None: - canonical = destination_strategy("google_cloud_logging") - assert canonical is not None - assert canonical.transport == "remote" - assert canonical.required_config == ("google_cloud_project",) - assert destination_strategy("google") is canonical - assert destination_strategy(" Google-Cloud ") is canonical - - -# ── Manager integration ────────────────────────────────────────────────── - - -def test_manager_breaker_never_disables_queued_destination() -> None: - inner = FailingDestination() - manager, failures = make_manager() - destination = QueuedLogDestination( - inner=inner, on_failure=manager.on_failure - ) - manager.destinations = [destination] - manager.configured = True - - for _ in range(5): - manager.emit({"event": "x"}, log_type="event", severity="INFO") - destination.close(5.0) - - operations = [op for op, *_ in failures] - assert destination in manager.destinations - assert "logging.destination_disabled" not in operations - assert "logging.destination_emit" not in operations - - -def test_manager_passes_queue_knobs_to_queued_destination( - fake_remote_strategy, -) -> None: - manager, _failures = make_manager( - ObservabilityConfig( - log_destinations=(fake_remote_strategy,), - log_queue_maxsize=50, - log_queue_close_timeout_seconds=7.5, - ) - ) - manager.configure() - - queued = manager.destinations[0] - assert queued.maxsize == 50 - assert queued.close_timeout_seconds == 7.5 - manager.close() - - -def test_reconfigure_closes_previous_destinations( - fake_remote_strategy, -) -> None: - manager, _failures = make_manager( - ObservabilityConfig(log_destinations=(fake_remote_strategy,)) - ) - manager.configure() - first = manager.destinations[0] - manager._consecutive_failures[id(first)] = 2 - - manager.configure() - - assert first._closed is True - assert manager._consecutive_failures == {} - assert manager.destinations[0] is not first - manager.close() - - -def test_reconfigure_after_close_resumes_delivery( - fake_remote_strategy, -) -> None: - """The fork/snapshot story: close → drops → configure() → a fresh - listener delivers again.""" - manager, _failures = make_manager( - ObservabilityConfig(log_destinations=(fake_remote_strategy,)) - ) - manager.configure() - manager.close() - manager.emit({"event": "dropped"}, log_type="event", severity="INFO") - - manager.configure() - manager.emit({"event": "delivered"}, log_type="event", severity="INFO") - manager.close(5.0) - - rebuilt = manager.destinations[0] - assert [p["event"] for p in rebuilt.inner.payloads] == ["delivered"] diff --git a/tests/test_release_scripts.py b/tests/test_release_scripts.py deleted file mode 100644 index 1ae568c..0000000 --- a/tests/test_release_scripts.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import annotations - -import importlib.util -from pathlib import Path - -import pytest - -REPO_ROOT = Path(__file__).resolve().parents[1] - - -def _load_script(path: str, module_name: str): - spec = importlib.util.spec_from_file_location( - module_name, - REPO_ROOT / path, - ) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module - - -bump_version = _load_script(".github/bump_version.py", "bump_version") -fetch_version = _load_script(".github/fetch_version.py", "fetch_version") - - -def test_infer_bump_uses_towncrier_fragment_precedence(tmp_path) -> None: - changelog_dir = tmp_path / "changelog.d" - changelog_dir.mkdir() - (changelog_dir / "feature.added.md").write_text("Add a feature.\n") - (changelog_dir / "api.breaking.md").write_text("Break an API.\n") - - assert bump_version.infer_bump(changelog_dir) == "major" - - -def test_infer_bump_rejects_missing_fragments(tmp_path) -> None: - changelog_dir = tmp_path / "changelog.d" - changelog_dir.mkdir() - - with pytest.raises(SystemExit): - bump_version.infer_bump(changelog_dir) - - -def test_current_version_prefers_highest_pyproject_changelog_or_tag( - tmp_path, - monkeypatch, -) -> None: - pyproject = tmp_path / "pyproject.toml" - pyproject.write_text('[project]\nversion = "0.1.0"\n') - changelog = tmp_path / "CHANGELOG.md" - changelog.write_text("## [0.2.0] - 2026-06-22\n") - monkeypatch.setattr( - bump_version, - "get_git_tag_versions", - lambda _repo_root: ["0.3.0"], - ) - - assert ( - bump_version.get_current_version(pyproject, changelog, tmp_path) - == "0.3.0" - ) - - -def test_update_file_replaces_project_version(tmp_path) -> None: - pyproject = tmp_path / "pyproject.toml" - pyproject.write_text('[project]\nversion = "0.1.0"\n') - - bump_version.update_file(pyproject, "0.2.0") - - assert 'version = "0.2.0"' in pyproject.read_text() - - -def test_fetch_version_reads_project_version(tmp_path) -> None: - pyproject = tmp_path / "pyproject.toml" - pyproject.write_text('[project]\nversion = "1.2.3"\n') - - assert fetch_version.fetch_version(pyproject) == "1.2.3" diff --git a/tests/test_runtime.py b/tests/test_runtime.py new file mode 100644 index 0000000..b61a913 --- /dev/null +++ b/tests/test_runtime.py @@ -0,0 +1,291 @@ +from __future__ import annotations + +import asyncio +import logging + +import pytest +from conftest import make_config, make_runtime, records + +from policyengine_observability import ( + REQUEST_ID_HEADER, + configure, + instrument_logging, +) + + +def test_operation_context_emits_one_completion(runtime) -> None: + observed, output = runtime + with observed.operation("simulation.run", attributes={"backend": "modal"}): + observed.event("simulation.started") + + emitted = records(output) + assert [item["event.name"] for item in emitted] == [ + "simulation.started", + "operation.completed", + ] + completion = emitted[-1] + assert completion["operation.name"] == "simulation.run" + assert completion["outcome"] == "success" + assert completion["attributes"] == {"backend": "modal"} + + +def test_span_does_not_emit_operation_completion(runtime) -> None: + observed, output = runtime + with observed.span("simulation.build"): + observed.event("inside") + assert [item["event.name"] for item in records(output)] == ["inside"] + + +def test_sync_decorator_preserves_result(runtime) -> None: + observed, output = runtime + + @observed.operation("calculate") + def calculate(value: int) -> int: + return value * 2 + + assert calculate(4) == 8 + assert records(output)[0]["event.name"] == "operation.completed" + + +def test_async_context_and_decorator_preserve_result(runtime) -> None: + observed, output = runtime + + @observed.span("child") + async def child() -> str: + await asyncio.sleep(0) + return "done" + + async def run() -> str: + async with observed.operation("async.run"): + return await child() + + assert asyncio.run(run()) == "done" + assert [item["event.name"] for item in records(output)] == [ + "operation.completed" + ] + + +def test_application_exception_is_same_object_when_export_fails( + runtime, +) -> None: + observed, _output = runtime + application_error = RuntimeError("application failure") + + def broken_emit(_record) -> None: + raise OSError("logging unavailable") + + observed._delivery.emit = broken_emit + + @observed.operation("broken") + def fail() -> None: + raise application_error + + with pytest.raises(RuntimeError) as caught: + fail() + assert caught.value is application_error + assert observed.diagnostics.count("failure.record.emit") == 1 + + +def test_observability_entry_failure_does_not_change_return( + monkeypatch, +) -> None: + observed, _output = make_runtime() + + def fail_start(*_args, **_kwargs): + raise RuntimeError("instrumentation failed") + + monkeypatch.setattr(observed, "_start_operation", fail_start) + + @observed.operation("work") + def work() -> int: + return 42 + + assert work() == 42 + assert observed.diagnostics.count("failure.operation.start") == 1 + observed.shutdown() + + +def test_process_control_exception_from_instrumentation_is_not_caught( + monkeypatch, +) -> None: + observed, _output = make_runtime() + + def stop(*_args, **_kwargs): + raise KeyboardInterrupt + + monkeypatch.setattr(observed, "_start_operation", stop) + with pytest.raises(KeyboardInterrupt): + with observed.operation("work"): + pass + observed.shutdown() + + +def test_request_accepts_valid_id_and_emits_once(runtime) -> None: + observed, output = runtime + request_id = observed.begin_request( + headers={REQUEST_ID_HEADER.lower(): "request-123"}, + method="get", + route="/household/", + ) + assert request_id == "request-123" + assert observed.response_headers()[REQUEST_ID_HEADER] == "request-123" + observed.end_request(status_code=200) + observed.end_request(status_code=200) + completion = records(output)[0] + assert completion["event.name"] == "request.completed" + assert completion["request.id"] == "request-123" + assert completion["http.request.method"] == "GET" + assert completion["http.route"] == "/household/" + assert completion["http.response.status_code"] == 200 + + +def test_request_replaces_malformed_id_and_classifies_status(runtime) -> None: + observed, output = runtime + request_id = observed.begin_request( + headers={REQUEST_ID_HEADER: "invalid id with spaces"}, + method="POST", + route="", + ) + assert request_id != "invalid id with spaces" + observed.update_request_route("/economy") + observed.end_request(status_code=404) + assert records(output)[0]["outcome"] == "client_error" + + +def test_request_exception_is_error(runtime) -> None: + observed, output = runtime + observed.begin_request(headers={}, method="GET", route="/fail") + error = ValueError("bad") + observed.record_exception(error, handled=False, status_code=500) + observed.end_request(status_code=500, error=error) + emitted = records(output) + assert emitted[0]["event.name"] == "exception.recorded" + assert emitted[1]["event.name"] == "request.completed" + assert emitted[1]["outcome"] == "error" + + +def test_set_context_and_capture_context_are_allowlisted(runtime) -> None: + observed, _output = runtime + observed.begin_request( + headers={REQUEST_ID_HEADER: "request-1"}, + method="POST", + route="/simulation", + ) + observed.set_context( + auth_result="accepted", + job_id="job-1", + household="must-not-appear", + unapproved="must-not-appear", + ) + captured = observed.capture_context() + assert captured["request_id"] == "request-1" + assert captured["job_id"] == "job-1" + assert "auth_result" not in captured + assert "household" not in captured + assert "unapproved" not in captured + assert "captured_at" in captured + assert observed.diagnostics.count("attributes.omitted") == 2 + observed.end_request(status_code=202) + + +def test_two_runtimes_keep_identity_and_context_separate() -> None: + first, first_output = make_runtime() + second_config = make_config( + service=make_config().service.__class__( + "second-api", "policyengine.api-v1", "1", "worker" + ) + ) + second = configure(second_config) + import io + + second_output = io.StringIO() + second._delivery._stdout = second_output + first.begin_request( + headers={REQUEST_ID_HEADER: "first-request"}, + method="GET", + route="/first", + ) + second.event("second.event") + first.end_request(status_code=200) + assert records(first_output)[0]["request.id"] == "first-request" + second_record = records(second_output)[0] + assert second_record["service.name"] == "second-api" + assert "request.id" not in second_record + first.shutdown() + second.shutdown() + + +def test_concurrent_async_tasks_do_not_share_request_ids() -> None: + observed, output = make_runtime() + + async def request(request_id: str) -> None: + observed.begin_request( + headers={REQUEST_ID_HEADER: request_id}, + method="GET", + route="/task", + ) + await asyncio.sleep(0) + observed.event("task.event") + observed.end_request(status_code=200) + + async def run() -> None: + await asyncio.gather(request("one"), request("two")) + + asyncio.run(run()) + grouped = { + item["request.id"] for item in records(output) if "request.id" in item + } + assert grouped == {"one", "two"} + observed.shutdown() + + +def test_standard_logging_is_explicit_and_idempotent(runtime) -> None: + observed, output = runtime + logger = logging.getLogger("tests.application") + logger.handlers.clear() + logger.propagate = False + first = instrument_logging(logger, observed) + second = instrument_logging(logger, observed) + assert first is second + logger.warning( + "structured warning", + extra={"policyengine_attributes": {"backend": "local"}}, + ) + item = records(output)[0] + assert item["message"] == "structured warning" + assert item["severity"] == "WARNING" + assert item["attributes"] == {"backend": "local"} + + +def test_shutdown_is_bounded_repeatable_and_restartable(runtime) -> None: + observed, _output = runtime + observed.shutdown() + observed.shutdown() + assert observed._closed + observed.restart_after_snapshot() + assert not observed._closed + + +def test_local_drop_and_export_diagnostics_increment_otel_metrics() -> None: + observed, _output = make_runtime() + calls: list[tuple[str, str, int]] = [] + + class Metrics: + def record_dropped(self, name: str, value: int) -> None: + calls.append(("dropped", name, value)) + + def record_exporter_failure(self, name: str, value: int) -> None: + calls.append(("failure", name, value)) + + def shutdown(self, _timeout: float) -> None: + pass + + observed._otel = Metrics() + observed.diagnostics.increment("logs.dropped.queue_full", 2) + observed.diagnostics.increment("logs.export_failure", 1) + observed.diagnostics.increment("unrelated", 1) + assert calls == [ + ("dropped", "logs.dropped.queue_full", 2), + ("failure", "logs.export_failure", 1), + ] + observed.shutdown() diff --git a/tests/test_runtime_emission.py b/tests/test_runtime_emission.py deleted file mode 100644 index 8a57f91..0000000 --- a/tests/test_runtime_emission.py +++ /dev/null @@ -1,425 +0,0 @@ -from __future__ import annotations - -from types import SimpleNamespace - -from runtime_helpers import ( - FailingInstrument, - FailingLogDestination, - RecordingInstrument, - RecordingLogDestination, - SegmentName, - runtime, -) - -from policyengine_observability import ( - ObservabilityConfig, - ObservabilityRuntime, - RequestObservabilityContext, - coerce_segment_name, -) -from policyengine_observability.destinations import ( - google_cloud_logging as google_cloud_logging_module, -) - - -def test_trace_helpers_log_failures_without_throwing() -> None: - observed = runtime() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - observed.propagate = SimpleNamespace( - inject=lambda _carrier: (_ for _ in ()).throw(RuntimeError("inject")), - extract=lambda _carrier: (_ for _ in ()).throw( - RuntimeError("extract") - ), - ) - observed.trace = SimpleNamespace( - get_current_span=lambda: (_ for _ in ()).throw(RuntimeError("span")) - ) - - assert observed.traceparent_header() is None - assert observed._extract_context({"traceparent": "parent"}) is None - assert observed._current_span() is None - assert failures == [ - "request.traceparent_header", - "otel.extract_context", - "otel.current_span", - ] - - -def test_record_event_covers_operation_context_and_no_context_metrics() -> ( - None -): - observed = runtime() - observed.failover_events = RecordingInstrument() - - with observed.operation("worker", flavor="queue"): - observed.record_event("modal_retry", attempt=1, ignored=None) - - observed.record_event("fallback_without_context") - - assert len(observed.failover_events.calls) == 2 - assert observed.failover_events.calls[0][2]["operation"] == "worker" - assert observed.failover_events.calls[1][2]["event"] == ( - "fallback_without_context" - ) - - -def test_record_event_request_context_and_emit_log_skip_paths() -> None: - observed = runtime() - observed.failover_events = RecordingInstrument() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/event", - path="/event", - endpoint="event", - query_keys=[], - content_length_bytes=None, - inbound={}, - internal_dispatch=True, - ) - observed.begin_request(context) - observed.record_event("fallback_request", detail="request") - observed.emit_request_log(context) - observed.emit_request_log(context) - observed.teardown_request(None) - - operation = observed.start_operation("job")["operation"] - observed.emit_operation_log(operation) - observed.emit_operation_log(operation) - observed.end_operation({"operation": operation}) - - assert observed.failover_events.calls[0][2]["route"] == "/event" - assert context.emitted is True - assert operation.emitted is True - - -def test_operation_log_emits_to_configured_destinations_once() -> None: - observed = runtime() - destination = RecordingLogDestination() - observed.log_destination_manager.destinations = [destination] - observed.log_destination_manager.configured = True - - handle = observed.start_operation("job") - operation = handle["operation"] - observed.end_operation(handle) - observed.emit_operation_log(operation) - - assert len(destination.calls) == 1 - payload, log_type, severity = destination.calls[0] - assert payload["operation"] == "job" - assert payload["severity"] == "INFO" - assert log_type == "operation" - assert severity == "INFO" - - -def test_request_log_emits_to_configured_destination_once() -> None: - observed = runtime() - destination = RecordingLogDestination() - observed.log_destination_manager.destinations = [destination] - observed.log_destination_manager.configured = True - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/calculate", - path="/calculate", - endpoint="calculate", - query_keys=[], - content_length_bytes=None, - inbound={"client_ip": "203.0.113.1"}, - status_code=200, - ) - - observed.emit_request_log(context) - observed.emit_request_log(context) - - assert len(destination.calls) == 1 - payload, log_type, severity = destination.calls[0] - assert payload["request_id"] == "request-1" - assert payload["client_ip"] == "203.0.113.1" - assert payload["severity"] == "INFO" - assert log_type == "request" - assert severity == "INFO" - - -def test_request_log_reserved_fields_override_inbound_and_attributes() -> None: - observed = runtime() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/calculate", - path="/calculate", - endpoint="calculate", - query_keys=[], - content_length_bytes=None, - inbound={ - "request_id": "inbound-request", - "status_code": "inbound-status", - "duration_ms": "inbound-duration", - "segment_tree": "inbound-tree", - }, - attributes={ - "request_id": "attribute-request", - "status_code": "attribute-status", - "duration_ms": "attribute-duration", - "timings_ms": "attribute-timings", - "timing_counts": "attribute-counts", - "segment_tree": "attribute-tree", - "error": "attribute-error", - }, - status_code=204, - ) - - payload = context.as_log_record(trace_id=None, span_id=None) - assert payload["request_id"] == "request-1" - assert payload["status_code"] == 204 - assert isinstance(payload["duration_ms"], float) - assert isinstance(payload["timings_ms"], dict) - assert isinstance(payload["timing_counts"], dict) - assert payload["segment_tree"] == [] - assert payload["error"] is None - - -def test_event_log_emits_to_configured_destination() -> None: - observed = runtime() - destination = RecordingLogDestination() - observed.log_destination_manager.destinations = [destination] - observed.log_destination_manager.configured = True - - observed.record_event("custom_event", detail="value") - - assert len(destination.calls) == 1 - payload, log_type, severity = destination.calls[0] - assert payload["event"] == "custom_event" - assert payload["detail"] == "value" - assert payload["service_name"] == "svc" - assert payload["severity"] == "INFO" - assert log_type == "event" - assert severity == "INFO" - - -def test_log_severity_maps_status_codes_and_errors() -> None: - observed = runtime() - - assert observed._severity_for_log_record({"status_code": 200}) == "INFO" - assert observed._severity_for_log_record({"status_code": 404}) == ( - "WARNING" - ) - assert observed._severity_for_log_record({"status_code": "500"}) == ( - "ERROR" - ) - assert ( - observed._severity_for_log_record({"error": {"handled": False}}) - == "ERROR" - ) - assert observed._severity_for_log_record({"error": {"handled": True}}) == ( - "WARNING" - ) - - -def test_destination_failure_logs_internal_error_without_throwing() -> None: - observed = runtime() - recording = RecordingLogDestination() - observed.log_destination_manager.destinations = [ - FailingLogDestination(), - recording, - ] - observed.log_destination_manager.configured = True - - with observed.operation("job"): - pass - - assert any( - payload["event"] == "observability_internal_error" - and payload["operation"] == "logging.destination_emit" - for payload, _log_type, _severity in recording.calls - ) - assert any( - payload.get("operation") == "job" - for payload, _log_type, _severity in recording.calls - ) - - -def test_all_destination_failures_fall_back_to_stderr(capsys) -> None: - observed = runtime() - observed.log_destination_manager.destinations = [FailingLogDestination()] - observed.log_destination_manager.configured = True - - with observed.operation("job"): - pass - - stderr = capsys.readouterr().err - assert "observability_internal_error" in stderr - assert "logging.destination_emit" in stderr - - -def test_unknown_destination_falls_back_to_stdout() -> None: - observed = ObservabilityRuntime( - ObservabilityConfig( - service_name="svc", - otel_enabled=False, - log_destinations=("missing",), - ) - ) - - observed.configure() - - assert [ - destination.name - for destination in observed.log_destination_manager.destinations - ] == ["stdout"] - - -def test_google_destination_init_failure_falls_back_to_stdout( - monkeypatch, -) -> None: - def fail_google_destination(**_kwargs): - raise ImportError("google-cloud-logging missing") - - monkeypatch.setattr( - google_cloud_logging_module, - "GoogleCloudLoggingDestination", - fail_google_destination, - ) - observed = ObservabilityRuntime( - ObservabilityConfig( - service_name="svc", - otel_enabled=False, - log_destinations=("google_cloud_logging",), - ) - ) - - observed.configure() - - assert [ - destination.name - for destination in observed.log_destination_manager.destinations - ] == ["stdout"] - - -def test_disabled_configure_does_not_initialize_log_destinations( - monkeypatch, -) -> None: - def fail_google_destination(**_kwargs): - raise AssertionError("google destination should not initialize") - - monkeypatch.setattr( - google_cloud_logging_module, - "GoogleCloudLoggingDestination", - fail_google_destination, - ) - observed = ObservabilityRuntime( - ObservabilityConfig( - service_name="svc", - enabled=False, - log_destinations=("google_cloud_logging",), - ) - ) - - observed.configure() - - assert observed.log_destination_manager.destinations == [] - assert observed.log_destination_manager.configured is False - - -def test_record_segment_metric_covers_calculation_and_backend() -> None: - observed = runtime() - observed.segment_duration = RecordingInstrument() - observed.calculate_duration = RecordingInstrument() - observed.backend_duration = RecordingInstrument() - - observed.record_segment_metric( - "calculation", - 0.1, - {"backend": "modal"}, - backend_segment=True, - ) - - assert observed.segment_duration.calls - assert observed.calculate_duration.calls - assert observed.backend_duration.calls - - -def test_metric_recording_failures_are_logged() -> None: - observed = runtime() - observed.operation_duration = FailingInstrument() - observed.http_duration = FailingInstrument() - observed.segment_duration = FailingInstrument() - observed.errors = FailingInstrument() - observed.rate_limited = FailingInstrument() - observed.failover_events = FailingInstrument() - observed.active_requests = FailingInstrument() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - - observed.record_operation_metric(0.1, {}) - observed.record_request_metric(0.1, {}) - observed.record_segment_metric("load", 0.1, {}) - observed.record_error_metric({}) - observed.record_rate_limited_metric({}) - observed.record_failover_event_metric({}) - observed.record_active_request(1, {}) - - assert failures == [ - "metrics.record_operation", - "metrics.record_request", - "metrics.record_segment", - "metrics.record_error", - "metrics.record_rate_limited", - "metrics.record_failover_event", - "metrics.add_active_request", - ] - - -def test_private_safety_helpers_cover_fallback_paths( - monkeypatch, - capsys, -) -> None: - observed = runtime() - - class Unprintable: - def __str__(self) -> str: - raise RuntimeError("cannot stringify") - - assert observed._safe_str(Unprintable()) == "" - assert observed._safe_traceback(Unprintable()) == "" - - original_dumps = __import__("json").dumps - - def failing_dumps(payload, *args, **kwargs): - if payload.get("event") == "bad": - raise RuntimeError("json failed") - return original_dumps(payload, *args, **kwargs) - - monkeypatch.setattr("json.dumps", failing_dumps) - assert "observability_internal_error" in observed._json({"event": "bad"}) - - monkeypatch.setattr( - "policyengine_observability.runtime.INTERNAL_LOGGER.error", - lambda _message: (_ for _ in ()).throw(RuntimeError("logger failed")), - ) - observed.log_observability_failure("test", RuntimeError("failed")) - assert "observability_internal_error" in capsys.readouterr().err - - -def test_coerce_segment_name_validates_registry() -> None: - assert coerce_segment_name(SegmentName.LOAD, registry=SegmentName) == ( - "load", - True, - ) - assert coerce_segment_name("other", registry=SegmentName) == ( - "other", - False, - ) - assert coerce_segment_name("load", registry=["load"]) == ("load", True) - assert coerce_segment_name(SegmentName.LOAD, registry=None) == ( - "load", - True, - ) diff --git a/tests/test_runtime_operations.py b/tests/test_runtime_operations.py deleted file mode 100644 index 13f8e4a..0000000 --- a/tests/test_runtime_operations.py +++ /dev/null @@ -1,295 +0,0 @@ -from __future__ import annotations - -import asyncio -from types import SimpleNamespace - -import pytest -from runtime_helpers import ( - RecordingInstrument, - RecordingSpan, - RecordingTracer, - SegmentName, - runtime, -) - -from policyengine_observability import ( - RequestObservabilityContext, -) -from policyengine_observability import _state as state_module - - -def test_operation_context_manager_async_and_exception_paths() -> None: - async def run() -> None: - observed = runtime() - observed.errors = RecordingInstrument() - - with pytest.raises(RuntimeError, match="async failed"): - async with observed.operation("async_job", flavor="worker"): - raise RuntimeError("async failed") - - assert observed.errors.calls[0][2]["error_type"] == "RuntimeError" - assert observed.current_operation() is None - - asyncio.run(run()) - - -def test_start_operation_with_parent_context_attaches_and_detaches() -> None: - observed = runtime() - observed.tracer = RecordingTracer() - parent_context = object() - - handle = observed.start_operation( - "parented", - parent_context=parent_context, - ) - observed.end_operation(handle) - - assert observed.tracer.calls[0][0] == "parented" - assert observed.current_operation() is None - - -def test_operation_attach_detach_and_reset_failures_are_logged( - monkeypatch, -) -> None: - from opentelemetry import context as otel_context - - observed = runtime() - observed.tracer = RecordingTracer() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append((operation, fields.get("token"))) - ) - monkeypatch.setattr( - otel_context, - "attach", - lambda _context: (_ for _ in ()).throw(RuntimeError("attach failed")), - ) - - handle = observed.start_operation("job", parent_context=object()) - observed.end_operation(handle) - - monkeypatch.setattr( - otel_context, - "detach", - lambda _token: (_ for _ in ()).throw(RuntimeError("detach failed")), - ) - observed.end_operation( - { - "operation": None, - "context_token": object(), - "timings_token": object(), - "start_token": object(), - "operation_token": object(), - } - ) - - assert ("operation.context_attach", None) in failures - assert ("operation.context_detach", None) in failures - assert ("operation.context_reset", "timings_token") in failures - assert ("operation.context_reset", "start_token") in failures - assert ("operation.context_reset", "operation_token") in failures - - -def test_operation_end_and_start_failures_are_logged(monkeypatch) -> None: - class BrokenVar: - def set(self, _value): - raise RuntimeError("set failed") - - observed = runtime() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - operation_handle = observed.start_operation("job") - operation = operation_handle["operation"] - operation.metric_recorded = True - observed.complete_operation(operation) - observed.complete_operation = lambda _operation: (_ for _ in ()).throw( - RuntimeError("complete failed") - ) - observed.end_operation(operation_handle) - - monkeypatch.setattr(state_module, "_OPERATION_CONTEXT", BrokenVar()) - handle = observed.start_operation("job") - - assert handle["operation"] is None - assert failures == ["operation.end", "operation.start"] - - -def test_standalone_segment_creates_implicit_operation_metrics() -> None: - observed = runtime() - observed.segment_duration = RecordingInstrument() - observed.operation_duration = RecordingInstrument() - observed.operations = RecordingInstrument() - emitted_payloads = [] - observed.emit_operation_log = lambda operation: emitted_payloads.append( - operation.as_log_record(trace_id=None, span_id=None) - ) - - with observed.segment(SegmentName.LOAD, flavor="cli", tool="loader"): - pass - - _, _, segment_attributes = observed.segment_duration.calls[0] - _, _, operation_attributes = observed.operation_duration.calls[0] - assert segment_attributes["operation"] == "load" - assert segment_attributes["flavor"] == "cli" - assert segment_attributes["tool"] == "loader" - assert operation_attributes["operation"] == "load" - assert operation_attributes["flavor"] == "cli" - assert emitted_payloads[0]["segment_tree"][0]["name"] == "load" - assert emitted_payloads[0]["segment_tree"][0]["attrs"] == { - "flavor": "cli", - "tool": "loader", - } - assert observed.current_operation() is None - - -def test_segment_with_request_context_does_not_create_implicit_operation( - monkeypatch, -) -> None: - observed = runtime() - observed.segment_duration = RecordingInstrument() - observed.operation_duration = RecordingInstrument() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/calculate", - path="/calculate", - endpoint="calculate", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - monkeypatch.setattr(observed, "current_context", lambda: context) - - with observed.segment(SegmentName.LOAD): - pass - - _, _, attributes = observed.segment_duration.calls[0] - assert attributes["route"] == "/calculate" - assert observed.operation_duration.calls == [] - - -def test_start_scope_outside_request_records_operation_segment_metrics() -> ( - None -): - observed = runtime() - observed.segment_duration = RecordingInstrument() - timings: dict[str, float] = {} - - handle = observed.start_scope( - timings, - name="chat_turn", - flavor="chat", - model="claude", - ) - with observed.segment(SegmentName.LOAD, tool="search"): - pass - observed.end_scope(handle) - - _, _, attributes = observed.segment_duration.calls[0] - assert "load_ms" in timings - assert attributes["operation"] == "chat_turn" - assert attributes["flavor"] == "chat" - assert attributes["model"] == "claude" - assert attributes["tool"] == "search" - - -def test_nested_scope_annotates_span_context_and_operation() -> None: - observed = runtime() - observed.tracer = RecordingTracer() - parent_context = object() - timings: dict[str, float] = {} - - with observed.operation("outer", flavor="chat"): - handle = observed.start_scope( - timings, - name="inner", - parent_context=parent_context, - ) - observed.annotate(handle, model="claude") - observed.mark("custom_ms", 1.23) - observed.mark_ttft() - observed.end_scope(handle) - - span = observed.tracer.span - assert span.attributes["model"] == "claude" - assert "custom_ms" in timings - - -def test_entrypoint_decorator_records_operation_metrics() -> None: - observed = runtime() - observed.operation_duration = RecordingInstrument() - observed.operations = RecordingInstrument() - - @observed.entrypoint("import_data", flavor="cli") - def run_import() -> str: - return "done" - - assert run_import() == "done" - _, _, attributes = observed.operation_duration.calls[0] - assert attributes["operation"] == "import_data" - assert attributes["flavor"] == "cli" - - -def test_async_segment_decorator_records_segment_metrics() -> None: - observed = runtime() - observed.segment_duration = RecordingInstrument() - - @observed.segment(SegmentName.SAVE, flavor="worker") - async def save() -> str: - return "saved" - - assert asyncio.run(save()) == "saved" - _, _, attributes = observed.segment_duration.calls[0] - assert attributes["operation"] == "save" - assert attributes["flavor"] == "worker" - - -def test_record_error_outside_request_uses_operation_context() -> None: - observed = runtime() - observed.errors = RecordingInstrument() - - with observed.operation("worker", flavor="queue"): - observed.record_error( - RuntimeError("failed"), - handled=True, - include_stack=False, - ) - - _, _, attributes = observed.errors.calls[0] - assert attributes["operation"] == "worker" - assert attributes["flavor"] == "queue" - assert attributes["error_type"] == "RuntimeError" - - -def test_record_error_on_request_updates_span_status() -> None: - observed = runtime() - span = RecordingSpan() - observed.trace = SimpleNamespace(get_current_span=lambda: span) - observed.StatusCode = SimpleNamespace(ERROR="ERROR") - observed.Status = lambda code, message: (code, message) - observed.errors = RecordingInstrument() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/error", - path="/error", - endpoint="error", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - observed.begin_request(context) - observed.record_error( - RuntimeError("failed"), - handled=True, - status_code=500, - ) - observed.teardown_request(None) - - assert span.exceptions - assert span.status == ("ERROR", "failed") diff --git a/tests/test_runtime_request_failures.py b/tests/test_runtime_request_failures.py deleted file mode 100644 index 8a22d57..0000000 --- a/tests/test_runtime_request_failures.py +++ /dev/null @@ -1,342 +0,0 @@ -from __future__ import annotations - -from runtime_helpers import ( - RecordingInstrument, - RecordingPropagator, - SegmentName, - runtime, -) - -from policyengine_observability import ( - ObservabilityConfig, - RequestObservabilityContext, -) -from policyengine_observability.config import DEFAULT_METRIC_ATTRIBUTE_KEYS - - -def test_prepare_response_includes_traceparent_when_available() -> None: - observed = runtime() - observed.propagate = RecordingPropagator() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/trace", - path="/trace", - endpoint="trace", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - observed.begin_request(context) - headers = observed.prepare_response(200) - observed.teardown_request(None) - - assert headers["traceparent"].startswith("00-4bf92f") - - -def test_rate_limited_request_records_rate_limit_metric() -> None: - observed = runtime() - observed.rate_limited = RecordingInstrument() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/limited", - path="/limited", - endpoint="limited", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - observed.begin_request(context) - headers = observed.finish_request(429) - observed.teardown_request(None) - - assert headers["X-PolicyEngine-Request-Id"] == "request-1" - assert context.attributes["rate_limited"] is True - assert observed.rate_limited.calls[0][0] == "add" - - -def test_teardown_request_records_unhandled_exception() -> None: - observed = runtime() - observed.errors = RecordingInstrument() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/error", - path="/error", - endpoint="error", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - observed.begin_request(context) - observed.teardown_request(RuntimeError("failed")) - - assert context.status_code == 500 - assert context.error is not None - assert context.error.handled is False - assert observed.errors.calls[0][0] == "add" - - -def test_from_env_invalid_shutdown_timeout_falls_back(monkeypatch) -> None: - monkeypatch.setenv("OBSERVABILITY_SHUTDOWN_TIMEOUT_SECONDS", "bad") - - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.shutdown_timeout_seconds == 3.0 - - -def test_from_env_reads_stdout_format(monkeypatch) -> None: - monkeypatch.setenv("OBSERVABILITY_STDOUT_FORMAT", "google") - - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.stdout_format == "google" - - -def test_from_env_reads_queue_knobs(monkeypatch) -> None: - monkeypatch.setenv("OBSERVABILITY_LOG_QUEUE_MAXSIZE", "50") - monkeypatch.setenv("OBSERVABILITY_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS", "1.5") - - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.log_queue_maxsize == 50 - assert config.log_queue_close_timeout_seconds == 1.5 - - -def test_from_env_queue_knobs_fall_back_on_garbage(monkeypatch) -> None: - monkeypatch.setenv("OBSERVABILITY_LOG_QUEUE_MAXSIZE", "many") - monkeypatch.setenv("OBSERVABILITY_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS", "soon") - - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.log_queue_maxsize == 1000 - assert config.log_queue_close_timeout_seconds == 2.0 - - -def test_from_env_enables_otel_by_default() -> None: - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.otel_enabled is True - - -def test_from_env_allows_otel_opt_out(monkeypatch) -> None: - monkeypatch.setenv("OTEL_ENABLED", "false") - - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.otel_enabled is False - - -def test_from_env_ignores_legacy_observability_otel_switch( - monkeypatch, -) -> None: - monkeypatch.setenv("OBSERVABILITY_OTEL_ENABLED", "false") - - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.otel_enabled is True - - -def test_from_env_reads_boolean_csv_and_environment(monkeypatch) -> None: - monkeypatch.setenv("OBSERVABILITY_SERVICE_NAME", "env-svc") - monkeypatch.setenv("DEPLOYMENT_ENVIRONMENT", "production") - monkeypatch.setenv("OBSERVABILITY_ENABLED", "off") - monkeypatch.setenv("OBSERVABILITY_REQUEST_LOGS_ENABLED", "false") - monkeypatch.setenv("OBSERVABILITY_LOG_RAW_IP", "0") - monkeypatch.setenv("OBSERVABILITY_LOG_LEVEL", "warning") - monkeypatch.setenv("OTEL_ENABLED", "1") - monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector") - monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf") - monkeypatch.setenv("OBSERVABILITY_TRACER_NAME", "tracer") - monkeypatch.setenv("OBSERVABILITY_METER_NAME", "meter") - monkeypatch.setenv( - "OBSERVABILITY_METRIC_ATTRIBUTE_KEYS", - "service.name, custom", - ) - monkeypatch.setenv( - "OBSERVABILITY_EXTRA_METRIC_ATTRIBUTE_KEYS", - "custom, other", - ) - - config = ObservabilityConfig.from_env( - service_name="svc", - instrument_fastapi=True, - instrument_httpx=True, - ) - - assert config.service_name == "env-svc" - assert config.environment == "production" - assert config.enabled is False - assert config.request_logs_enabled is False - assert config.log_raw_ip is False - assert config.otel_enabled is True - assert config.otlp_endpoint == "http://collector" - assert config.otlp_protocol == "http/protobuf" - assert config.tracer_name == "tracer" - assert config.meter_name == "meter" - assert config.instrument_fastapi is True - assert config.instrument_httpx is True - assert config.metric_attribute_keys == ("service.name", "custom", "other") - - -def test_from_env_reads_log_destinations_and_google_config( - monkeypatch, -) -> None: - monkeypatch.setenv( - "OBSERVABILITY_LOG_DESTINATIONS", - "stdout, google-cloud-logging, stdout", - ) - monkeypatch.setenv("GCP_PROJECT", "fallback-project") - monkeypatch.setenv("OBSERVABILITY_GOOGLE_CLOUD_LOG_NAME", "custom-log") - - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.log_destinations == ("stdout", "google-cloud-logging") - assert config.google_cloud_project == "fallback-project" - assert config.google_cloud_log_name == "custom-log" - - -def test_from_env_uses_default_log_destinations_without_env( - monkeypatch, -) -> None: - monkeypatch.delenv("OBSERVABILITY_LOG_DESTINATIONS", raising=False) - - config = ObservabilityConfig.from_env( - service_name="svc", - default_log_destinations=("google_cloud_logging",), - ) - - assert config.log_destinations == ("google_cloud_logging",) - - -def test_from_env_log_destinations_env_overrides_default( - monkeypatch, -) -> None: - monkeypatch.setenv("OBSERVABILITY_LOG_DESTINATIONS", "stdout") - - config = ObservabilityConfig.from_env( - service_name="svc", - default_log_destinations=("google_cloud_logging",), - ) - - assert config.log_destinations == ("stdout",) - - -def test_metric_attribute_keys_are_configurable() -> None: - config = ObservabilityConfig( - service_name="svc", - metric_attribute_keys=("service.name", "tool"), - ) - context = RequestObservabilityContext( - config=config, - request_id="request-1", - method="POST", - route="/chat", - path="/chat", - endpoint="chat", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - context.set_attribute("tool", "search") - context.set_attribute("model", "claude") - - assert context.metric_attributes() == { - "service.name": "svc", - "tool": "search", - } - - -def test_context_set_attribute_normalizes_enum_values() -> None: - observed = runtime() - operation = observed.start_operation("job")["operation"] - request = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/", - path="/", - endpoint="root", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - operation.set_attribute("segment", SegmentName.LOAD) - request.set_attribute("segment", SegmentName.SAVE) - observed.end_operation({"operation": operation}) - - assert operation.attributes["segment"] == "load" - assert request.attributes["segment"] == "save" - - -def test_metric_attribute_keys_can_be_extended_from_env(monkeypatch) -> None: - monkeypatch.setenv("OBSERVABILITY_EXTRA_METRIC_ATTRIBUTE_KEYS", "custom") - - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.metric_attribute_keys == ( - *DEFAULT_METRIC_ATTRIBUTE_KEYS, - "custom", - ) - - -def test_segment_metric_uses_configured_metric_attribute_keys() -> None: - observed = runtime( - metric_attribute_keys=( - "service.name", - "route", - "method", - "segment", - "tool", - ) - ) - observed.segment_duration = RecordingInstrument() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="POST", - route="/chat", - path="/chat", - endpoint="chat", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - observed.begin_request(context) - with observed.segment(SegmentName.LOAD, tool="search", model="claude"): - pass - observed.finish_request(200) - observed.teardown_request(None) - - _, _, attributes = observed.segment_duration.calls[0] - assert attributes["tool"] == "search" - assert "model" not in attributes - - -def test_shutdown_calls_trace_and_metric_providers() -> None: - class Provider: - def __init__(self) -> None: - self.shutdown_called = False - - def shutdown(self) -> None: - self.shutdown_called = True - - observed = runtime(shutdown_timeout_seconds=1) - trace_provider = Provider() - meter_provider = Provider() - observed.tracer_provider = trace_provider - observed.meter_provider = meter_provider - - observed.shutdown() - - assert trace_provider.shutdown_called - assert meter_provider.shutdown_called diff --git a/tests/test_runtime_requests.py b/tests/test_runtime_requests.py deleted file mode 100644 index fb42d62..0000000 --- a/tests/test_runtime_requests.py +++ /dev/null @@ -1,331 +0,0 @@ -from __future__ import annotations - -from contextvars import ContextVar -from types import SimpleNamespace - -from runtime_helpers import ( - NamedRecordingSpan, - RecordingInstrument, - SegmentName, - runtime, -) - -from policyengine_observability import ( - RequestObservabilityContext, -) -from policyengine_observability import _state as state_module - - -def test_request_lifecycle_records_headers_and_context_metrics() -> None: - observed = runtime() - observed.active_requests = RecordingInstrument() - observed.requests = RecordingInstrument() - observed.http_duration = RecordingInstrument() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/calculate", - path="/calculate", - endpoint="calculate", - query_keys=["country"], - content_length_bytes=None, - inbound={"ip_source": "remote_addr", "client_ip": "127.0.0.1"}, - ) - - observed.begin_request(context) - with observed.segment(SegmentName.LOAD): - pass - headers = observed.finish_request(200) - observed.teardown_request(None) - - assert headers["X-PolicyEngine-Request-Id"] == "request-1" - assert context.status_code == 200 - assert "load" in context.timings_ms - assert observed.current_context() is None - assert observed.current_operation() is None - assert observed.active_requests.calls[0][1] == 1 - assert observed.active_requests.calls[-1][1] == -1 - assert observed.requests.calls[0][1] == 1 - - -def test_request_log_accumulates_repeated_segment_timings() -> None: - observed = runtime() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/calculate", - path="/calculate", - endpoint="calculate", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - observed.begin_request(context) - with observed.segment(SegmentName.LOAD): - pass - with observed.segment(SegmentName.LOAD): - pass - observed.finish_request(200) - observed.teardown_request(None) - - assert context.timings_ms["load"] >= 0 - assert context.timing_counts["load"] == 2 - payload = context.as_log_record(trace_id=None, span_id=None) - assert payload["timing_counts"]["load"] == 2 - assert [node["name"] for node in payload["segment_tree"]] == [ - "load", - "load", - ] - - -def test_internal_dispatch_segments_merge_into_parent_operation() -> None: - observed = runtime() - handle = observed.start_operation( - "modal_worker_dispatch", - flavor="modal_worker", - ) - parent_operation = handle["operation"] - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="POST", - route="/calculate", - path="/calculate", - endpoint="calculate", - query_keys=[], - content_length_bytes=None, - inbound={}, - internal_dispatch=True, - ) - - try: - observed.begin_request(context) - with observed.segment(SegmentName.LOAD): - pass - observed.finish_request(200) - observed.teardown_request(None) - - assert context.timings_ms is parent_operation.timings_ms - assert context.timing_counts is parent_operation.timing_counts - assert context.segment_tree is parent_operation.segment_tree - assert "load" in parent_operation.timings_ms - assert parent_operation.timing_counts["load"] == 1 - assert parent_operation.segment_tree[0].name == "load" - assert observed.current_operation() is parent_operation - finally: - observed.end_operation(handle) - - assert observed.current_context() is None - assert observed.current_operation() is None - - -def test_non_internal_request_timings_do_not_leak_to_parent_operation() -> ( - None -): - observed = runtime() - handle = observed.start_operation("job", flavor="worker") - parent_operation = handle["operation"] - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="POST", - route="/calculate", - path="/calculate", - endpoint="calculate", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - try: - observed.begin_request(context) - with observed.segment(SegmentName.LOAD): - pass - observed.finish_request(200) - observed.teardown_request(None) - - assert context.timings_ms is not parent_operation.timings_ms - assert context.segment_tree is not parent_operation.segment_tree - assert "load" not in parent_operation.timings_ms - assert parent_operation.segment_tree == [] - assert context.segment_tree[0].name == "load" - assert observed.current_operation() is parent_operation - finally: - observed.end_operation(handle) - - assert observed.current_context() is None - assert observed.current_operation() is None - - -def test_set_attribute_updates_explicit_operation_inside_request() -> None: - observed = runtime() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/chat", - path="/chat", - endpoint="chat", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - observed.begin_request(context) - handle = observed.start_operation("chat.turn", flavor="chat") - operation = handle["operation"] - try: - observed.set_attribute("model", "claude") - finally: - observed.end_operation(handle) - observed.teardown_request(None) - - assert context.attributes["model"] == "claude" - assert context.operation_context.attributes["model"] == "claude" - assert operation.attributes["model"] == "claude" - assert observed.current_context() is None - assert observed.current_operation() is None - - -def test_mark_ttft_attribute_updates_current_operation() -> None: - observed = runtime() - handle = observed.start_operation("chat.turn", flavor="chat") - operation = handle["operation"] - - try: - observed.mark_ttft_attribute() - finally: - observed.end_operation(handle) - - assert operation.attributes["ttft_ms"] >= 0 - - -def test_request_methods_noop_without_current_context() -> None: - observed = runtime() - - assert observed.prepare_response(200) == {} - observed.complete_request(200) - observed.update_request_route(route="/missing") - observed.teardown_request(None) - - -def test_request_begin_operation_begin_and_lifecycle_failures_are_logged( - monkeypatch, -) -> None: - class BrokenVar: - def set(self, _value): - raise RuntimeError("set failed") - - observed = runtime() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/broken", - path="/broken", - endpoint="broken", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - monkeypatch.setattr(state_module, "_REQUEST_CONTEXT", BrokenVar()) - observed.begin_request(context) - monkeypatch.setattr( - state_module, - "_REQUEST_CONTEXT", - ContextVar("request", default=None), - ) - monkeypatch.setattr(state_module, "_OPERATION_CONTEXT", BrokenVar()) - observed._begin_request_operation(context) - - assert failures == ["request.begin", "request.operation_begin"] - - -def test_request_prepare_complete_update_and_teardown_failures_are_logged() -> ( - None -): - observed = runtime() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/broken", - path="/broken", - endpoint="broken", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - observed.begin_request(context) - context.span_attributes = lambda **_extra: (_ for _ in ()).throw( - RuntimeError("span attrs failed") - ) - observed.prepare_response(200) - observed.complete_request(200) - observed.update_request_route(route="/other") - observed.emit_request_log = lambda _context: (_ for _ in ()).throw( - RuntimeError("emit failed") - ) - observed.teardown_request(None) - - assert "request.prepare_response" in failures - assert "request.complete" in failures - assert "request.update_route" in failures - assert "request.teardown" in failures - - -def test_set_attribute_failure_path_is_logged() -> None: - observed = runtime() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - observed.current_context = lambda: SimpleNamespace( - set_attribute=lambda *_args: (_ for _ in ()).throw( - RuntimeError("attribute failed") - ) - ) - - observed.set_attribute("tool", "loader") - - assert failures == ["request.set_attribute"] - - -def test_request_route_update_relabels_active_request_and_span() -> None: - observed = runtime() - observed.active_requests = RecordingInstrument() - span = NamedRecordingSpan() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/initial", - path="/items/1", - endpoint="initial", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - context.server_span = span - - observed.begin_request(context) - observed.update_request_route(route="/items/", endpoint="item") - observed.teardown_request(None) - - assert context.route == "/items/" - assert context.endpoint == "item" - assert span.names == ["/items/"] - assert observed.active_requests.calls[1][1] == -1 - assert observed.active_requests.calls[2][1] == 1 diff --git a/tests/test_runtime_segments.py b/tests/test_runtime_segments.py deleted file mode 100644 index c92021a..0000000 --- a/tests/test_runtime_segments.py +++ /dev/null @@ -1,303 +0,0 @@ -from __future__ import annotations - -import asyncio -from typing import Any - -import pytest -from runtime_helpers import ( - AttributeFailingSpan, - RecordingSpan, - RecordingTracer, - SegmentName, - runtime, -) - -from policyengine_observability import ( - UNKNOWN_SEGMENT, - ObservabilityRuntime, - RequestObservabilityContext, -) - - -def test_segment_records_aggregated_timing() -> None: - observed = runtime() - - with observed.collect_timings("request") as timings: - with observed.segment(SegmentName.LOAD): - pass - with observed.segment(SegmentName.LOAD): - pass - - assert "load_ms" in timings - assert timings["load_ms"] >= 0 - - -def test_operation_log_accumulates_repeated_segment_timings() -> None: - observed = runtime() - handle = observed.start_operation("job") - operation = handle["operation"] - - try: - with observed.segment(SegmentName.LOAD): - pass - with observed.segment(SegmentName.LOAD): - pass - finally: - observed.end_operation(handle) - - assert operation.timings_ms["load"] >= 0 - assert operation.timing_counts["load"] == 2 - payload = operation.as_log_record(trace_id=None, span_id=None) - assert payload["timing_counts"]["load"] == 2 - - -def test_operation_log_records_ordered_nested_segment_tree() -> None: - observed = runtime() - handle = observed.start_operation("job") - operation = handle["operation"] - - try: - with observed.segment(SegmentName.LOAD): - with observed.segment( - SegmentName.SAVE, - simulation_kind="baseline", - token="SECRET", - payload={"not": "safe"}, - ): - pass - with observed.segment( - SegmentName.SAVE, - simulation_kind="reform", - ): - pass - finally: - observed.end_operation(handle) - - payload = operation.as_log_record(trace_id=None, span_id=None) - tree = payload["segment_tree"] - assert len(tree) == 1 - assert tree[0]["sequence"] == 1 - assert tree[0]["name"] == "load" - assert "duration_ms" in tree[0] - assert "self_ms" not in tree[0] - - children = tree[0]["children"] - assert [child["sequence"] for child in children] == [2, 3] - assert [child["name"] for child in children] == ["save", "save"] - assert children[0]["attrs"] == {"simulation_kind": "baseline"} - assert children[1]["attrs"] == {"simulation_kind": "reform"} - assert "token" not in children[0].get("attrs", {}) - assert "payload" not in children[0].get("attrs", {}) - assert payload["timing_counts"]["save"] == 2 - - -def test_operation_log_reserved_fields_override_attributes() -> None: - observed = runtime() - handle = observed.start_operation( - "job", - operation="attribute-operation", - duration_ms="attribute-duration", - timings_ms="attribute-timings", - timing_counts="attribute-counts", - segment_tree="attribute-tree", - error="attribute-error", - ) - operation = handle["operation"] - - try: - with observed.segment(SegmentName.LOAD): - pass - finally: - observed.end_operation(handle) - - payload = operation.as_log_record(trace_id=None, span_id=None) - assert payload["operation"] == "job" - assert isinstance(payload["duration_ms"], float) - assert isinstance(payload["timings_ms"], dict) - assert isinstance(payload["timing_counts"], dict) - assert isinstance(payload["segment_tree"], list) - assert payload["error"] is None - - -def test_async_segment_records_timing() -> None: - async def run() -> dict[str, float]: - observed = runtime() - with observed.collect_timings("request") as timings: - async with observed.asegment(SegmentName.SAVE): - pass - return timings - - timings = asyncio.run(run()) - - assert "save_ms" in timings - - -def test_async_segments_keep_independent_segment_tree_stacks() -> None: - async def run() -> list[dict[str, Any]]: - observed = runtime() - handle = observed.start_operation("job") - operation = handle["operation"] - - async def branch(branch_name: str) -> None: - async with observed.asegment(SegmentName.LOAD, branch=branch_name): - await asyncio.sleep(0) - async with observed.asegment( - SegmentName.SAVE, - branch=branch_name, - ): - await asyncio.sleep(0) - - try: - await asyncio.gather(branch("a"), branch("b")) - finally: - observed.end_operation(handle) - return operation.as_log_record(trace_id=None, span_id=None)[ - "segment_tree" - ] - - tree = asyncio.run(run()) - - assert [node["name"] for node in tree] == ["load", "load"] - assert [node["attrs"] for node in tree] == [ - {"branch": "a"}, - {"branch": "b"}, - ] - assert [node["children"][0]["attrs"] for node in tree] == [ - {"branch": "a"}, - {"branch": "b"}, - ] - - -def test_segment_preserves_business_exception_and_records_timing() -> None: - observed = runtime() - - with pytest.raises(ValueError, match="business failed"): - with observed.collect_timings("request") as timings: - with observed.segment(SegmentName.LOAD): - raise ValueError("business failed") - - assert "load_ms" in timings - - -def test_segment_tree_records_failed_segments_before_reraising() -> None: - observed = runtime() - handle = observed.start_operation("job") - operation = handle["operation"] - error = None - - try: - with observed.segment(SegmentName.LOAD): - raise ValueError("business failed") - except ValueError as exc: - error = exc - finally: - observed.end_operation(handle, error) - - payload = operation.as_log_record(trace_id=None, span_id=None) - assert payload["event"] == "operation_failed" - assert payload["segment_tree"][0]["name"] == "load" - assert "duration_ms" in payload["segment_tree"][0] - - -def test_unregistered_segment_falls_back_without_throwing() -> None: - class BrokenString: - def __str__(self) -> str: - raise RuntimeError("cannot stringify") - - observed = runtime() - - with observed.collect_timings("request") as timings: - with observed.segment(BrokenString()): - pass - - assert f"{UNKNOWN_SEGMENT}_ms" in timings - - -def test_segment_span_start_failure_does_not_skip_user_code() -> None: - observed = runtime() - observed.tracer = RecordingTracer(fail_enter=True) - executed = False - - with observed.collect_timings("request") as timings: - with observed.segment(SegmentName.LOAD): - executed = True - - assert executed - assert "load_ms" in timings - - -def test_segment_span_exit_failure_does_not_escape() -> None: - observed = runtime() - observed.tracer = RecordingTracer(fail_exit=True) - - with observed.collect_timings("request") as timings: - with observed.segment(SegmentName.LOAD): - pass - - assert "load_ms" in timings - - -def test_disabled_runtime_noops_across_public_methods() -> None: - observed = ObservabilityRuntime.disabled() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/disabled", - path="/disabled", - endpoint="disabled", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - handle = observed.start_operation("disabled") - observed.end_operation(handle) - observed.begin_request(context) - observed.complete_request(200) - observed.update_request_route(route="/other") - observed.teardown_request(None) - observed.set_attribute("key", "value") - observed.record_error(RuntimeError("ignored"), handled=True) - observed.record_event("ignored") - - with observed.segment(SegmentName.LOAD) as span: - assert span is None - - async def run() -> None: - async with observed.asegment(SegmentName.LOAD) as async_span: - assert async_span is None - - asyncio.run(run()) - assert observed.prepare_response(200) == {} - assert observed.current_context() is None - assert observed.current_operation() is None - - -def test_span_attribute_failure_does_not_drop_span_lifecycle() -> None: - observed = runtime() - span = AttributeFailingSpan() - observed.tracer = RecordingTracer(span=span) - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - - with observed.segment(SegmentName.LOAD, tool="loader"): - pass - - assert "otel.span_attributes" in failures - assert observed.tracer.last_context_manager.exited - - -def test_collect_timings_records_block_exception_on_scope_span() -> None: - observed = runtime() - span = RecordingSpan() - observed.tracer = RecordingTracer(span=span) - - with pytest.raises(RuntimeError, match="scope failed"): - with observed.collect_timings("turn"): - raise RuntimeError("scope failed") - - assert len(span.exceptions) == 1 - assert isinstance(span.exceptions[0], RuntimeError) diff --git a/tests/test_runtime_tracing.py b/tests/test_runtime_tracing.py deleted file mode 100644 index cef5cf3..0000000 --- a/tests/test_runtime_tracing.py +++ /dev/null @@ -1,420 +0,0 @@ -from __future__ import annotations - -import builtins -import time -from types import SimpleNamespace - -import pytest -from runtime_helpers import ( - AttributeFailingSpan, - ExceptionFailingSpan, - RecordingMeter, - RecordingPropagator, - RecordingTracer, - ValidContextSpan, - runtime, -) - -from policyengine_observability import ( - RequestObservabilityContext, -) -from policyengine_observability import _state as state_module -from policyengine_observability import runtime as runtime_module - - -def test_shutdown_logs_provider_failures_and_timeout() -> None: - class FailingProvider: - def shutdown(self) -> None: - raise RuntimeError("shutdown failed") - - class SlowProvider: - def shutdown(self) -> None: - time.sleep(0.05) - - observed = runtime(shutdown_timeout_seconds=0.001) - observed.tracer_provider = FailingProvider() - observed.meter_provider = SlowProvider() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - - observed.shutdown() - - assert "otel.trace_shutdown" in failures - assert "otel.shutdown_timeout" in failures - - -def test_shutdown_closes_destinations_with_full_budget_and_no_thread( - monkeypatch, -) -> None: - observed = runtime(shutdown_timeout_seconds=2.0) - close_calls = [] - monkeypatch.setattr( - observed.log_destination_manager, - "close", - lambda deadline=None: close_calls.append(deadline), - ) - - def fail_thread(*args, **kwargs): - raise AssertionError( - "no watchdog thread should exist without providers" - ) - - monkeypatch.setattr(runtime_module.threading, "Thread", fail_thread) - - observed.shutdown() - - assert close_calls == [2.0] - - -def test_shutdown_destination_deadline_fits_inside_provider_budget( - monkeypatch, -) -> None: - """The prior design gave the log flush a deadline larger than the - join bounding it, starving provider shutdown; the deadline must be - derived from (and smaller than) the shutdown budget.""" - - class Provider: - def __init__(self) -> None: - self.shutdown_called = False - - def shutdown(self) -> None: - self.shutdown_called = True - - observed = runtime(shutdown_timeout_seconds=2.0) - provider = Provider() - observed.tracer_provider = provider - close_calls = [] - monkeypatch.setattr( - observed.log_destination_manager, - "close", - lambda deadline=None: close_calls.append(deadline), - ) - - observed.shutdown() - - assert close_calls == [1.0] - assert provider.shutdown_called - - -def test_shutdown_slow_destination_close_still_runs_providers() -> None: - class Provider: - def __init__(self) -> None: - self.shutdown_called = False - - def shutdown(self) -> None: - self.shutdown_called = True - - observed = runtime(shutdown_timeout_seconds=0.2) - provider = Provider() - observed.tracer_provider = provider - observed.log_destination_manager.close = lambda deadline=None: time.sleep( - 0.05 - ) - - observed.shutdown() - - assert provider.shutdown_called - - -def test_shutdown_clamps_pathological_budget(monkeypatch) -> None: - observed = runtime(shutdown_timeout_seconds=float("inf")) - close_calls = [] - monkeypatch.setattr( - observed.log_destination_manager, - "close", - lambda deadline=None: close_calls.append(deadline), - ) - - observed.shutdown() - - assert close_calls == [3.0] - - -def test_restart_log_destinations_rebuilds_from_config() -> None: - observed = runtime(otel_enabled=False) - observed.configure() - first = observed.log_destination_manager.destinations[0] - - observed.restart_log_destinations() - - rebuilt = observed.log_destination_manager.destinations - assert len(rebuilt) == 1 - assert rebuilt[0] is not first - assert observed.log_destination_manager.configured is True - - -def test_restart_log_destinations_noops_when_disabled(monkeypatch) -> None: - """The kill switch must hold across forks and snapshot restores: - a disabled runtime's restart must not build destinations.""" - observed = runtime(enabled=False) - configure_calls = [] - monkeypatch.setattr( - observed.log_destination_manager, - "configure", - lambda: configure_calls.append(True), - ) - - observed.restart_log_destinations() - - assert configure_calls == [] - - -def test_shutdown_survives_destination_close_failure(monkeypatch) -> None: - class Provider: - def __init__(self) -> None: - self.shutdown_called = False - - def shutdown(self) -> None: - self.shutdown_called = True - - observed = runtime(shutdown_timeout_seconds=1.0) - provider = Provider() - observed.tracer_provider = provider - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - - def broken_close(deadline=None): - raise RuntimeError("close exploded") - - monkeypatch.setattr( - observed.log_destination_manager, "close", broken_close - ) - - observed.shutdown() - - assert provider.shutdown_called - assert "logging.destination_close" in failures - - -def test_configure_otel_creates_real_providers_and_instruments() -> None: - observed = runtime(otel_enabled=True) - - observed.configure() - - assert observed.tracer is not None - assert observed.meter is not None - assert observed.trace is not None - assert observed.propagate is not None - - -def test_configure_otel_with_exporters_does_not_throw() -> None: - observed = runtime( - otel_enabled=True, - otlp_endpoint="http://localhost:4318", - otlp_protocol="http/protobuf", - ) - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - - observed.configure() - - assert observed.tracer_provider is not None - assert observed.meter_provider is not None - - -def test_configure_otel_import_failure_is_logged(monkeypatch) -> None: - observed = runtime(otel_enabled=True) - failures = [] - original_import = builtins.__import__ - - def failing_import(name, *args, **kwargs): - if name == "opentelemetry": - raise RuntimeError("otel missing") - return original_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", failing_import) - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - - observed.configure() - - assert failures == ["otel.configure_imports"] - - -def test_configure_instruments_and_instrument_failures() -> None: - observed = runtime() - meter = RecordingMeter() - observed.meter = meter - - observed._configure_instruments() - - assert len(meter.created) == 11 - - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append((operation, fields.get("instrument"))) - ) - noop = observed._instrument( - lambda *_args, **_kwargs: (_ for _ in ()).throw( - RuntimeError("factory failed") - ), - "broken", - ) - - noop.add(1) - noop.record(1) - assert failures == [("metrics.create_instrument", "broken")] - - -def test_request_span_lifecycle_records_enter_and_exit_failures() -> None: - observed = runtime() - observed.tracer = RecordingTracer(fail_enter=True) - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/span", - path="/span", - endpoint="span", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - observed._start_request_span(context) - assert context.server_span is None - - observed.tracer = RecordingTracer(fail_exit=True) - observed._start_request_span(context) - observed._close_request_span(context, RuntimeError("failed")) - observed._close_request_span(context, None) - - assert failures == ["otel.request_span_enter", "otel.request_span_exit"] - - -def test_safe_span_records_exception_and_preserves_user_error() -> None: - observed = runtime() - observed.tracer = RecordingTracer() - - with pytest.raises(RuntimeError, match="business failed"): - with observed._safe_span("safe", {}): - raise RuntimeError("business failed") - - assert isinstance(observed.tracer.span.exceptions[0], RuntimeError) - - -def test_span_and_segment_failure_helpers_are_logged(monkeypatch) -> None: - observed = runtime() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - observed.trace = SimpleNamespace( - get_current_span=lambda: AttributeFailingSpan() - ) - observed._set_current_span_attributes({"key": "value"}) - - observed.trace = SimpleNamespace( - get_current_span=lambda: ExceptionFailingSpan() - ) - observed._record_exception_on_span( - ExceptionFailingSpan(), - RuntimeError("failed"), - handled=False, - status_code=500, - ) - observed._add_span_event("event", {"safe": "yes", "unsafe": object()}) - observed._record_segment_safely("missing_start", None, {}) - monkeypatch.setattr( - observed, - "_safe_perf_counter", - lambda _operation: None, - ) - observed._record_segment_safely("missing_end", 1.0, {}) - - assert "otel.set_span_attributes" in failures - assert "otel.record_exception" in failures - - -def test_segment_helpers_cover_operation_attrs_and_span_prefix() -> None: - observed = runtime(span_prefix="svc") - with observed.operation("job", flavor="cli"): - attrs = observed._segment_span_attributes({"tool": "loader"}) - - assert attrs["policyengine.operation"] == "job" - assert attrs["tool"] == "loader" - assert observed._span_name("load") == "svc.load" - - -def test_contextvar_failure_paths_are_logged(monkeypatch) -> None: - class BrokenVar: - def get(self): - raise RuntimeError("get failed") - - observed = runtime() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - monkeypatch.setattr(state_module, "_REQUEST_CONTEXT", BrokenVar()) - monkeypatch.setattr(state_module, "_OPERATION_CONTEXT", BrokenVar()) - - assert observed.current_context() is None - assert observed.current_operation() is None - assert failures == ["context.current", "operation.current"] - - -def test_runtime_owned_httpx_instrumentation_failure_does_not_throw( - monkeypatch, -) -> None: - observed = runtime(otel_enabled=True) - failures = [] - original_import = builtins.__import__ - - def failing_import(name, *args, **kwargs): - if name == "opentelemetry.instrumentation.httpx": - raise RuntimeError("instrumentation failed") - return original_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", failing_import) - monkeypatch.setattr( - observed, - "log_observability_failure", - lambda operation, exc, **fields: failures.append(operation), - ) - - observed.instrument_httpx() - - assert failures == ["httpx.auto_instrument"] - - -def test_runtime_owned_httpx_instrumentation_success_and_wrapper() -> None: - from policyengine_observability.integrations.httpx import ( - instrument_httpx, - ) - - observed = runtime(otel_enabled=True) - - instrument_httpx(observed) - instrument_httpx(observed) - - assert observed._httpx_instrumented is True - - -def test_traceparent_capture_and_valid_trace_ids() -> None: - observed = runtime() - propagator = RecordingPropagator() - observed.propagate = propagator - span = ValidContextSpan() - observed.trace = SimpleNamespace(get_current_span=lambda: span) - - trace_id, span_id = observed._trace_ids() - - assert observed.traceparent_header().startswith("00-4bf92f") - assert observed._extract_context({"traceparent": "parent"}) == { - "parent": {"traceparent": "parent"} - } - assert propagator.extracted == {"traceparent": "parent"} - assert trace_id == "4bf92f3577b34da6a3ce929d0e0e4736" - assert span_id == "00f067aa0ba902b7" diff --git a/tests/test_stdout_destinations.py b/tests/test_stdout_destinations.py deleted file mode 100644 index aef9c4c..0000000 --- a/tests/test_stdout_destinations.py +++ /dev/null @@ -1,362 +0,0 @@ -from __future__ import annotations - -import json - -from fakes import ( - ClosableRecordingDestination, - FailingDestination, - RecordingDestination, - RecordingLogger, - make_manager, -) - -from policyengine_observability.config import ObservabilityConfig -from policyengine_observability.destinations.stdout import ( - StdoutJsonDestination, - resolve_stdout_formatter, -) - - -def _stdout_destination(config=None, formatter=None): - logger = RecordingLogger() - if formatter is None and config is not None: - formatter = resolve_stdout_formatter(config) - destination = StdoutJsonDestination( - loggers={"event": logger}, - serializer=json.dumps, - formatter=formatter, - ) - return destination, logger - - -def _emitted_line(logger): - ((_, message),) = logger.lines - return json.loads(message) - - -def test_stdout_google_formatter_maps_agent_native_keys() -> None: - config = ObservabilityConfig( - stdout_format="google", google_cloud_project="proj" - ) - destination, logger = _stdout_destination(config) - - destination.emit( - { - "schema_version": "policyengine.observability.event.v1", - "service_name": "svc", - "event": "x", - "trace_id": "abc123", - "span_id": 456, - }, - log_type="event", - severity="error", - ) - - line = _emitted_line(logger) - assert line["severity"] == "ERROR" - assert line["logging.googleapis.com/trace"] == ( - "projects/proj/traces/abc123" - ) - assert line["logging.googleapis.com/spanId"] == "456" - assert line["logging.googleapis.com/labels"] == { - "log_type": "event", - "service_name": "svc", - "schema_version": "policyengine.observability.event.v1", - } - assert "time" not in line - assert line["event"] == "x" - - -def test_stdout_google_formatter_omits_trace_without_project() -> None: - config = ObservabilityConfig(stdout_format="google") - destination, logger = _stdout_destination(config) - - destination.emit( - {"event": "x", "trace_id": "abc"}, log_type="event", severity="INFO" - ) - - line = _emitted_line(logger) - assert "logging.googleapis.com/trace" not in line - - -def test_stdout_unknown_format_falls_back_to_plain() -> None: - config = ObservabilityConfig(stdout_format=" GoOgLeX ") - destination, logger = _stdout_destination(config) - - destination.emit({"event": "x"}, log_type="event", severity="INFO") - - assert _emitted_line(logger) == {"event": "x"} - - -def test_stdout_unknown_format_reports_when_channel_available() -> None: - failures = [] - formatter = resolve_stdout_formatter( - ObservabilityConfig(stdout_format="agent-natve"), - on_failure=lambda operation, exc, **fields: failures.append( - (operation, str(exc)) - ), - ) - - formatted = formatter({"event": "x"}, log_type="event", severity="INFO") - - assert formatted == {"event": "x"} - assert len(failures) == 1 - assert failures[0][0] == "logging.stdout_format" - assert "agent-natve" in failures[0][1] - - -def test_stdout_format_name_is_normalized() -> None: - config = ObservabilityConfig(stdout_format=" GOOGLE ") - destination, logger = _stdout_destination(config) - - destination.emit({"event": "x"}, log_type="event", severity="INFO") - - assert _emitted_line(logger)["severity"] == "INFO" - - -def test_stdout_broken_formatter_degrades_to_unformatted() -> None: - def broken(payload, *, log_type, severity): - raise RuntimeError("formatter bug") - - destination, logger = _stdout_destination(formatter=broken) - - destination.emit({"event": "x"}, log_type="event", severity="INFO") - - assert _emitted_line(logger) == {"event": "x"} - - -def test_stdout_broken_formatter_factory_degrades_to_plain() -> None: - from policyengine_observability.destinations.stdout import ( - _FORMATTER_FACTORIES, - register_stdout_formatter, - ) - - def broken_factory(config): - raise RuntimeError("factory bug") - - register_stdout_formatter("broken-test", broken_factory) - try: - failures = [] - formatter = resolve_stdout_formatter( - ObservabilityConfig(stdout_format="broken-test"), - on_failure=lambda operation, exc, **fields: failures.append( - (operation, fields) - ), - ) - finally: - _FORMATTER_FACTORIES.pop("broken_test", None) - - formatted = formatter({"event": "x"}, log_type="event", severity="INFO") - - assert formatted == {"event": "x"} - assert len(failures) == 1 - assert failures[0][0] == "logging.stdout_format" - assert failures[0][1]["stdout_format"] == "broken-test" - - -def test_configure_fallback_survives_broken_formatter_factory() -> None: - """The crash path: no destination builds, so the manager's - last-resort stdout fallback resolves the same broken formatter — - configure must stay fail-open and emit must still write plain.""" - from policyengine_observability.destinations.stdout import ( - _FORMATTER_FACTORIES, - register_stdout_formatter, - ) - - def broken_factory(config): - raise RuntimeError("factory bug") - - register_stdout_formatter("broken-test", broken_factory) - try: - config = ObservabilityConfig( - log_destinations=("nonexistent",), stdout_format="broken-test" - ) - logger = RecordingLogger() - manager, _failures = make_manager(config, loggers={"event": logger}) - - manager.configure() # raised before the resolver guard existed - manager.emit({"event": "x"}, log_type="event", severity="INFO") - finally: - _FORMATTER_FACTORIES.pop("broken_test", None) - - assert _emitted_line(logger) == {"event": "x", "severity": "INFO"} - - -def test_stdout_google_formatter_never_mutates_caller_payload() -> None: - config = ObservabilityConfig( - stdout_format="google", google_cloud_project="proj" - ) - destination, logger = _stdout_destination(config) - payload = {"event": "x", "trace_id": "abc"} - - destination.emit(payload, log_type="event", severity="INFO") - - assert payload == {"event": "x", "trace_id": "abc"} - - -def test_custom_stdout_formatter_registers_and_resolves() -> None: - from policyengine_observability.destinations.stdout import ( - _FORMATTER_FACTORIES, - register_stdout_formatter, - ) - - def factory(config): - return lambda payload, *, log_type, severity: {"wrapped": payload} - - register_stdout_formatter("custom-test", factory) - try: - # Hyphen/underscore variance is forgiven the same way it is for - # destination names. - config = ObservabilityConfig(stdout_format=" Custom_Test ") - destination, logger = _stdout_destination(config) - destination.emit({"event": "x"}, log_type="event", severity="INFO") - finally: - _FORMATTER_FACTORIES.pop("custom_test", None) - - assert _emitted_line(logger) == {"wrapped": {"event": "x"}} - - -# ── Destination circuit breaker and manager lifecycle ─────────────────── - - -def test_destination_disabled_after_consecutive_emit_failures() -> None: - from policyengine_observability.destinations.manager import ( - DESTINATION_FAILURE_LIMIT, - ) - - flaky = FailingDestination() - healthy = RecordingDestination() - manager, failures = make_manager(destinations=[flaky, healthy]) - - for _ in range(DESTINATION_FAILURE_LIMIT + 2): - manager.emit({"event": "x"}, log_type="event", severity="INFO") - - assert flaky.calls == DESTINATION_FAILURE_LIMIT - assert flaky not in manager.destinations - assert len(healthy.payloads) == DESTINATION_FAILURE_LIMIT + 2 - assert any(op == "logging.destination_disabled" for op, *_ in failures) - - -def test_emit_success_resets_the_failure_counter() -> None: - from policyengine_observability.destinations.manager import ( - DESTINATION_FAILURE_LIMIT, - ) - - flaky = FailingDestination(fail_first=DESTINATION_FAILURE_LIMIT - 1) - manager, failures = make_manager(destinations=[flaky]) - - for _ in range(DESTINATION_FAILURE_LIMIT + 2): - manager.emit({"event": "x"}, log_type="event", severity="INFO") - - assert flaky in manager.destinations - counts = [ - fields["consecutive_failures"] - for op, _exc, fields in failures - if op == "logging.destination_emit" - ] - assert max(counts) == DESTINATION_FAILURE_LIMIT - 1 - - -def test_sole_disabled_destination_falls_back_to_stdout() -> None: - from policyengine_observability.destinations.manager import ( - DESTINATION_FAILURE_LIMIT, - ) - - flaky = FailingDestination() - manager, _failures = make_manager(destinations=[flaky]) - - for _ in range(DESTINATION_FAILURE_LIMIT + 1): - manager.emit({"event": "x"}, log_type="event", severity="INFO") - - assert flaky not in manager.destinations - assert any( - isinstance(destination, StdoutJsonDestination) - for destination in manager.destinations - ) - - -def test_manager_close_reports_failures_and_closes_the_rest() -> None: - class ExplodingClose(RecordingDestination): - def close(self) -> None: - raise RuntimeError("close failed") - - exploding = ExplodingClose() - closable = ClosableRecordingDestination() - manager, failures = make_manager(destinations=[exploding, closable]) - - manager.close(1.0) # must not raise - - assert closable.closed == 1 - assert any(op == "logging.destination_close" for op, *_ in failures) - - -def test_manager_close_accepts_zero_argument_close() -> None: - """A duck-typed close(self) works under every close path — the - deadline is passed only when the signature accepts it.""" - closable = ClosableRecordingDestination() - manager, failures = make_manager(destinations=[closable]) - - manager.close(1.0) - - assert closable.closed == 1 - assert failures == [] - - -def test_manager_close_shares_one_deadline_across_destinations() -> None: - import time - - deadlines = [] - - class SlowClose(RecordingDestination): - def close(self, deadline_seconds=None) -> None: - deadlines.append(deadline_seconds) - time.sleep(0.05) - - manager, _failures = make_manager(destinations=[SlowClose(), SlowClose()]) - - manager.close(1.0) - - first, second = deadlines - # The second destination only gets what the first one left, so N - # stuck destinations cannot take N times the budget. - assert first <= 1.0 - assert second <= first - 0.04 - - -def test_emit_falls_back_to_stdout_when_configure_crashes( - monkeypatch, -) -> None: - logger = RecordingLogger() - manager, failures = make_manager(loggers={"event": logger}) - - def broken_configure() -> None: - raise RuntimeError("configure exploded") - - monkeypatch.setattr(manager, "configure", broken_configure) - - manager.emit({"event": "x"}, log_type="event", severity="INFO") - - assert manager.configured is True - assert isinstance(manager.destinations[0], StdoutJsonDestination) - assert _emitted_line(logger) == {"event": "x", "severity": "INFO"} - assert any(op == "logging.destination_config" for op, *_ in failures) - - -def test_reconfigure_closes_previous_after_installing_new() -> None: - """Close-phase failure reports route through emit; the replaced - destinations must be closed only after the new ones are installed - so those reports still have a sink.""" - sink_states = [] - manager, _failures = make_manager() - - class CloseProbe(RecordingDestination): - def close(self) -> None: - sink_states.append(list(manager.destinations)) - - manager.destinations = [CloseProbe()] - manager.configured = True - - manager.configure() - - assert len(sink_states) == 1 - assert sink_states[0], "previous destination closed before new install" diff --git a/uv.lock b/uv.lock index 08efba2..a292ab2 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,6 @@ version = 1 revision = 3 -requires-python = ">=3.12" +requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.14'", "python_full_version == '3.13.*'", @@ -38,15 +38,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/16/9826f089383c593cdfc4a6e5aca94d9e91ae1692c57af82c3b2aa5e810f7/anyio-4.14.0-py3-none-any.whl", hash = "sha256:dd9b7a2a9799ed6552fde617b2c5df02b7fdd7d88392fc48101e51bae46164d9", size = 123506, upload-time = "2026-06-15T22:00:47.595Z" }, ] -[[package]] -name = "asgiref" -version = "3.11.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/63/40/f03da1264ae8f7cfdbf9146542e5e7e8100a4c66ab48e791df9a03d3f6c0/asgiref-3.11.1.tar.gz", hash = "sha256:5f184dc43b7e763efe848065441eac62229c9f7b0475f41f80e207a114eda4ce", size = 38550, upload-time = "2026-02-03T13:30:14.33Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/0a/a72d10ed65068e115044937873362e6e32fab1b7dce0046aeb224682c989/asgiref-3.11.1-py3-none-any.whl", hash = "sha256:e8667a091e69529631969fd45dc268fa79b99c92c5fcdda727757e52146ec133", size = 24345, upload-time = "2026-02-03T13:30:13.039Z" }, -] - [[package]] name = "blinker" version = "1.9.0" @@ -88,6 +79,19 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/12/4a/3dfd5f7850cbf0d06dc84ba9aa00db766b52ca38d8b86e3a38314d52498c/cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", size = 184344, upload-time = "2025-09-08T23:22:26.456Z" }, + { url = "https://files.pythonhosted.org/packages/4f/8b/f0e4c441227ba756aafbe78f117485b25bb26b1c059d01f137fa6d14896b/cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", size = 180560, upload-time = "2025-09-08T23:22:28.197Z" }, + { url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613, upload-time = "2025-09-08T23:22:29.475Z" }, + { url = "https://files.pythonhosted.org/packages/b8/56/6033f5e86e8cc9bb629f0077ba71679508bdf54a9a5e112a3c0b91870332/cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", size = 216476, upload-time = "2025-09-08T23:22:31.063Z" }, + { url = "https://files.pythonhosted.org/packages/dc/7f/55fecd70f7ece178db2f26128ec41430d8720f2d12ca97bf8f0a628207d5/cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", size = 203374, upload-time = "2025-09-08T23:22:32.507Z" }, + { url = "https://files.pythonhosted.org/packages/84/ef/a7b77c8bdc0f77adc3b46888f1ad54be8f3b7821697a7b89126e829e676a/cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", size = 202597, upload-time = "2025-09-08T23:22:34.132Z" }, + { url = "https://files.pythonhosted.org/packages/d7/91/500d892b2bf36529a75b77958edfcd5ad8e2ce4064ce2ecfeab2125d72d1/cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", size = 215574, upload-time = "2025-09-08T23:22:35.443Z" }, + { url = "https://files.pythonhosted.org/packages/44/64/58f6255b62b101093d5df22dcb752596066c7e89dd725e0afaed242a61be/cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", size = 218971, upload-time = "2025-09-08T23:22:36.805Z" }, + { url = "https://files.pythonhosted.org/packages/ab/49/fa72cebe2fd8a55fbe14956f9970fe8eb1ac59e5df042f603ef7c8ba0adc/cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", size = 211972, upload-time = "2025-09-08T23:22:38.436Z" }, + { url = "https://files.pythonhosted.org/packages/0b/28/dd0967a76aab36731b6ebfe64dec4e981aff7e0608f60c2d46b46982607d/cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", size = 217078, upload-time = "2025-09-08T23:22:39.776Z" }, + { url = "https://files.pythonhosted.org/packages/2b/c0/015b25184413d7ab0a410775fdb4a50fca20f5589b5dab1dbbfa3baad8ce/cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", size = 172076, upload-time = "2025-09-08T23:22:40.95Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8f/dc5531155e7070361eb1b7e4c1a9d896d0cb21c49f807a6c03fd63fc877e/cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", size = 182820, upload-time = "2025-09-08T23:22:42.463Z" }, + { url = "https://files.pythonhosted.org/packages/95/5c/1b493356429f9aecfd56bc171285a4c4ac8697f76e9bbbbb105e537853a1/cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", size = 177635, upload-time = "2025-09-08T23:22:43.623Z" }, { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, @@ -142,6 +146,22 @@ version = "3.4.7" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/d7/b5b7020a0565c2e9fa8c09f4b5fa6232feb326b8c20081ccded47ea368fd/charset_normalizer-3.4.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7", size = 309705, upload-time = "2026-04-02T09:26:02.191Z" }, + { url = "https://files.pythonhosted.org/packages/5a/53/58c29116c340e5456724ecd2fff4196d236b98f3da97b404bc5e51ac3493/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7", size = 206419, upload-time = "2026-04-02T09:26:03.583Z" }, + { url = "https://files.pythonhosted.org/packages/b2/02/e8146dc6591a37a00e5144c63f29fb7c97a734ea8a111190783c0e60ab63/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e", size = 227901, upload-time = "2026-04-02T09:26:04.738Z" }, + { url = "https://files.pythonhosted.org/packages/fb/73/77486c4cd58f1267bf17db420e930c9afa1b3be3fe8c8b8ebbebc9624359/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c", size = 222742, upload-time = "2026-04-02T09:26:06.36Z" }, + { url = "https://files.pythonhosted.org/packages/a1/fa/f74eb381a7d94ded44739e9d94de18dc5edc9c17fb8c11f0a6890696c0a9/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df", size = 214061, upload-time = "2026-04-02T09:26:08.347Z" }, + { url = "https://files.pythonhosted.org/packages/dc/92/42bd3cefcf7687253fb86694b45f37b733c97f59af3724f356fa92b8c344/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265", size = 199239, upload-time = "2026-04-02T09:26:09.823Z" }, + { url = "https://files.pythonhosted.org/packages/4c/3d/069e7184e2aa3b3cddc700e3dd267413dc259854adc3380421c805c6a17d/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4", size = 210173, upload-time = "2026-04-02T09:26:10.953Z" }, + { url = "https://files.pythonhosted.org/packages/62/51/9d56feb5f2e7074c46f93e0ebdbe61f0848ee246e2f0d89f8e20b89ebb8f/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e", size = 209841, upload-time = "2026-04-02T09:26:12.142Z" }, + { url = "https://files.pythonhosted.org/packages/d2/59/893d8f99cc4c837dda1fe2f1139079703deb9f321aabcb032355de13b6c7/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38", size = 200304, upload-time = "2026-04-02T09:26:13.711Z" }, + { url = "https://files.pythonhosted.org/packages/7d/1d/ee6f3be3464247578d1ed5c46de545ccc3d3ff933695395c402c21fa6b77/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c", size = 229455, upload-time = "2026-04-02T09:26:14.941Z" }, + { url = "https://files.pythonhosted.org/packages/54/bb/8fb0a946296ea96a488928bdce8ef99023998c48e4713af533e9bb98ef07/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b", size = 210036, upload-time = "2026-04-02T09:26:16.478Z" }, + { url = "https://files.pythonhosted.org/packages/9a/bc/015b2387f913749f82afd4fcba07846d05b6d784dd16123cb66860e0237d/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c", size = 224739, upload-time = "2026-04-02T09:26:17.751Z" }, + { url = "https://files.pythonhosted.org/packages/17/ab/63133691f56baae417493cba6b7c641571a2130eb7bceba6773367ab9ec5/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d", size = 216277, upload-time = "2026-04-02T09:26:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/06/6d/3be70e827977f20db77c12a97e6a9f973631a45b8d186c084527e53e77a4/charset_normalizer-3.4.7-cp311-cp311-win32.whl", hash = "sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad", size = 147819, upload-time = "2026-04-02T09:26:20.295Z" }, + { url = "https://files.pythonhosted.org/packages/20/d9/5f67790f06b735d7c7637171bbfd89882ad67201891b7275e51116ed8207/charset_normalizer-3.4.7-cp311-cp311-win_amd64.whl", hash = "sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00", size = 159281, upload-time = "2026-04-02T09:26:21.74Z" }, + { url = "https://files.pythonhosted.org/packages/ca/83/6413f36c5a34afead88ce6f66684d943d91f233d76dd083798f9602b75ae/charset_normalizer-3.4.7-cp311-cp311-win_arm64.whl", hash = "sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1", size = 147843, upload-time = "2026-04-02T09:26:22.901Z" }, { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328, upload-time = "2026-04-02T09:26:24.331Z" }, { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061, upload-time = "2026-04-02T09:26:25.568Z" }, { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031, upload-time = "2026-04-02T09:26:26.865Z" }, @@ -236,6 +256,21 @@ version = "7.14.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/9c/a3/3834a5564fe8f32154cd7032400d3c2f9c565b2a373fa671f2bbdad6f634/coverage-7.14.2.tar.gz", hash = "sha256:7a2da3d81cfe17c18038c6d98e6592aa9147d596d056119b0ee612c3c8bd5230", size = 923982, upload-time = "2026-06-20T14:49:30.885Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/04/d5/d0e511247f84fa88ae7da68403cbd3bf9d2a5fc48f5d6618a6846b275632/coverage-7.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:909f265c8c41f04c824bf741b2601fdcb56cab4bf56e018996b6494192ba0f58", size = 220352, upload-time = "2026-06-20T14:47:28.61Z" }, + { url = "https://files.pythonhosted.org/packages/03/4a/ecaff6db72e6c1782ca51336e391393f1e9cc6e4412d6c3da8b7d5075adf/coverage-7.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c8102deaf911938233f760426e6a5e287388521de95111d5c8de26c8a1028924", size = 220855, upload-time = "2026-06-20T14:47:29.972Z" }, + { url = "https://files.pythonhosted.org/packages/34/9a/cf950cd8e8df06ee5941276e69f81647005360421be523d5ca18f658e143/coverage-7.14.2-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:851f49e7bd7d1cdaf328f3133942b252d5e3d3380690131f423cba8e435b87f5", size = 251276, upload-time = "2026-06-20T14:47:31.413Z" }, + { url = "https://files.pythonhosted.org/packages/9d/08/f973be32c9a095e4bb2d3a7bdcb2f9c117e39d4062471ffffae3623f6c51/coverage-7.14.2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:04cb445bed86aaf00aaa97d41a8b6e30f100f21e81c34caaec4efc684cb57768", size = 253189, upload-time = "2026-06-20T14:47:32.727Z" }, + { url = "https://files.pythonhosted.org/packages/96/aa/f3a50952ba553d442d94b793e5dede25d426b02e5e011e9a9dd225c002d3/coverage-7.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7471bc920d97c51c37ea8127f13b2adca43c3d78c53313b26a1f428e99d2c254", size = 255299, upload-time = "2026-06-20T14:47:34.019Z" }, + { url = "https://files.pythonhosted.org/packages/e0/29/9a4c491986f4d637ed64961ae56721661fc21b6b767d280848d0c708756a/coverage-7.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:da5057e1bb257c967feee8ba67f3ebf379e801c7717f238b3d8c9caf00fc8f93", size = 257255, upload-time = "2026-06-20T14:47:35.397Z" }, + { url = "https://files.pythonhosted.org/packages/dd/61/d2a5b48007f6a212f321c36cf5486feb80505d2d00dfb1163aad2da71197/coverage-7.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33c0da852e8a40246cd8e20cf3b2fc17ca52a45e9b5f7983c93db26f5d24b87b", size = 251417, upload-time = "2026-06-20T14:47:36.677Z" }, + { url = "https://files.pythonhosted.org/packages/ea/25/8df66ae25b401d4529e1d0617af20d9695d171ea4ffec4ca9dffc5dc37b7/coverage-7.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:f48a85bb437fab7782021c40bfee6b15146928b96960d008ace41b6901a0f21d", size = 252991, upload-time = "2026-06-20T14:47:38.027Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7b/16bdc9116dd8bf412a421a7227daa65ad9f12bef0685b13c1bd1c12e6d4c/coverage-7.14.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f44e7579a769a21d5b5e3166916bfe30ee175aaffff750324cbb11be2dbec5ad", size = 251051, upload-time = "2026-06-20T14:47:39.26Z" }, + { url = "https://files.pythonhosted.org/packages/0a/f8/b7dbed84274dcc69ddb9c0fe72ec1260830473e0d6c299dcf087a0567f7c/coverage-7.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:78853ca3c6ca2f012daa2b07dbabbb8db0f09d4dbe8ee828d294b3445d3f4cd8", size = 254817, upload-time = "2026-06-20T14:47:40.995Z" }, + { url = "https://files.pythonhosted.org/packages/c6/07/4659e6bed01a25a0effb4952e8e75fd157038fe5f2829b0f69c6811c2033/coverage-7.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:c9c2795ee3692097ff226ab806005d36bb9691fca9b35353542b57ea749cc830", size = 250772, upload-time = "2026-06-20T14:47:42.306Z" }, + { url = "https://files.pythonhosted.org/packages/26/f4/45019da4cd6cd1df3042476447449d62a76a201f6b3556aa40ac31bce20b/coverage-7.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2f5cc48a845d755b6db236f8c29c2b54773eb4c7e4ee2ead43812d73718784b0", size = 251679, upload-time = "2026-06-20T14:47:43.703Z" }, + { url = "https://files.pythonhosted.org/packages/92/e5/76d75fa2ffe0285d3f2608d1bb241fc245cf98fe614d52118427dd6ccdaa/coverage-7.14.2-cp311-cp311-win32.whl", hash = "sha256:9c61cb7eaabcfa609c5bc0f5ff5869d72a2f02f17994e5fba5f971de516f3c82", size = 222445, upload-time = "2026-06-20T14:47:45.137Z" }, + { url = "https://files.pythonhosted.org/packages/57/59/696c64547e5c8b9ed31532e9c7a5f9b6474054da93f8ab07f8baf7365c57/coverage-7.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:e715909b0966d1774d8a26e14e2f4a3ae75909dca526901c6306286b2dcbfbdc", size = 222922, upload-time = "2026-06-20T14:47:46.67Z" }, + { url = "https://files.pythonhosted.org/packages/63/72/646a28100462996c11b98e27d6786cd61f48100d1479804846a3e1e5bf9b/coverage-7.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:9193f7150937a4fd836b10eaa123e15d98e961d1fabac07e60adf2d4785f888a", size = 222468, upload-time = "2026-06-20T14:47:48.119Z" }, { url = "https://files.pythonhosted.org/packages/d0/d9/bdd141aa2c605096a8ef63b8435fd4f5fec78946a3cb7b9145840ec78291/coverage-7.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:37c94712e533ea06f0b1e4d934811c520b1914ce0e4da3916220717aa7a86bc6", size = 220528, upload-time = "2026-06-20T14:47:49.652Z" }, { url = "https://files.pythonhosted.org/packages/02/97/d24ae7d2afc62c54a36313d4dedb655c9afbba3003f0f7f1ae81e97af31f/coverage-7.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c050bbc7bba94c77e4ed7438f4fda1babe98ab145691d80aa6f60df934a1468b", size = 220883, upload-time = "2026-06-20T14:47:51.036Z" }, { url = "https://files.pythonhosted.org/packages/f8/0e/d8f00efd3df0d63e6843ebcbade9e4119d60f5376753c9705d84b014c775/coverage-7.14.2-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a7af571767a2ee342a171c16fc1b1a07a0bf511606d381703fb7cf397fe49d46", size = 252395, upload-time = "2026-06-20T14:47:52.627Z" }, @@ -347,6 +382,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6c/a0/db537264e234f7273a73ec020873d6d6b39dfd8a53db78b550ca8320440e/cryptography-49.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e", size = 4834820, upload-time = "2026-06-12T20:01:51.847Z" }, { url = "https://files.pythonhosted.org/packages/93/77/8df9eb486495979bccecd1062e2eaf435250e84437040295b57d09048b0b/cryptography-49.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b", size = 4967968, upload-time = "2026-06-12T20:02:12.524Z" }, { url = "https://files.pythonhosted.org/packages/c2/e6/f60198ea8d9dfa15fff9ed4ca02ce362f6eadd9ba757dcc50634c4257b63/cryptography-49.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001", size = 3785547, upload-time = "2026-06-12T20:02:26.847Z" }, + { url = "https://files.pythonhosted.org/packages/63/d3/4a83af35d65e3fad632c926fad684c193ea4398569ccb0bbbc7fe8f5dc9a/cryptography-49.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc1e275c2f1d97b1a6450b8b0ea3ebfa6e087a611c2b26cb2404d48588abab7b", size = 3993685, upload-time = "2026-06-12T20:02:14.883Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a7/f9dac0ab7f80368c56993a7bf638ef9935f825c91902798481fac0898138/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c83782480a4a9da4d0feb51950131ba32e12e70813848b3343f6e18c28a66838", size = 4676239, upload-time = "2026-06-12T20:02:28.793Z" }, + { url = "https://files.pythonhosted.org/packages/d7/70/2ba3769dd0ae167e2f33dfa9592d45db6ff9a61d62ca1a5b3d1bdd09068f/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b39efa323140595abd3ecca8529d321ae50f55f3aa3ba9cc81ea56a6011953d5", size = 4715584, upload-time = "2026-06-12T20:01:27.495Z" }, + { url = "https://files.pythonhosted.org/packages/94/64/2923570ac1c0bd3a737aa366ac3abbbbde273042308b8cde95e2364a6e6a/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:b47db11c2c3525083296069b98ac5221907455e989ae0c2e3008bde851921615", size = 4675885, upload-time = "2026-06-12T20:01:55.49Z" }, + { url = "https://files.pythonhosted.org/packages/ab/f8/614dc7e051418cfe53d55173c1e24c6b0085e89996fe90508c2fdf769aef/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:084ef1af862eb07ec46d25f68689f2102a9fc0e05ce7b80f14f5fe51e4eef0f6", size = 4715449, upload-time = "2026-06-12T20:02:05.469Z" }, + { url = "https://files.pythonhosted.org/packages/aa/50/a9caea39ad19c431c1a3f8a31114df65b260cdfe67786b6c7e7c040c4c44/cryptography-49.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6", size = 3783731, upload-time = "2026-06-12T20:02:43.319Z" }, ] [[package]] @@ -520,6 +561,16 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/b0/b5/1ff353970a87eda4c98251e34d2dfd214abd4982dc89119c9252a2a482d2/grpcio-1.81.1.tar.gz", hash = "sha256:6fa10a767143a5e82e8eaab53918af0cd8909a57a27f8cb2288b80a613ac671b", size = 13026582, upload-time = "2026-06-11T12:46:51.673Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/52/ea/1c2fa386b718ff493225e61cfc052ef400b4d6ffc54cbe261026432624b5/grpcio-1.81.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:d71d30f2d92f67d944631c523713934fee37292469e182ebcd2c1dd8a64ce53f", size = 6093112, upload-time = "2026-06-11T12:44:52.131Z" }, + { url = "https://files.pythonhosted.org/packages/2b/18/acf45fa8bd1bc5d7b0c2fd3dc4c209379fbd5bb396b440b68a83342226b7/grpcio-1.81.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b137f4bf3ada9dc44d411478decc6ff09a79ed30b306cd2abaa98408c3588137", size = 12074277, upload-time = "2026-06-11T12:44:55.354Z" }, + { url = "https://files.pythonhosted.org/packages/48/d7/ee86a60699b7db039f772a2c4a7e4facc7138984ff42c0130933a0063884/grpcio-1.81.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a3acb384427816dd5d470f47e62137b87f74da694faa8a50147012cf40df276a", size = 6640348, upload-time = "2026-06-11T12:44:59.223Z" }, + { url = "https://files.pythonhosted.org/packages/26/ee/d2de5e47378ffc207d476c230fea3be4d2601edbce9995f4fe45535d4896/grpcio-1.81.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:f9a0ebbe45c29b5e5866593c12b78bd9035f0f0f0d4bc8361680cd580d99db49", size = 7331842, upload-time = "2026-06-11T12:45:02.001Z" }, + { url = "https://files.pythonhosted.org/packages/23/d6/abeda5c2b896a0b341584fe5ac411bbf72e197a9a374c355fb90965e08d2/grpcio-1.81.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a37165cc80b1a368384b383e63a4c38116a10467ae44c904d2d7468c4470ec2", size = 6842229, upload-time = "2026-06-11T12:45:04.76Z" }, + { url = "https://files.pythonhosted.org/packages/10/1c/1f0da7d590b4aeee006826ba568d0e419ca14b23e18f901a3da3e9fba613/grpcio-1.81.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6282caffb41ec326d4cb67ca9cf53b739d1b2f975a2acb498c7418e9f7d9a416", size = 7446096, upload-time = "2026-06-11T12:45:07.499Z" }, + { url = "https://files.pythonhosted.org/packages/6a/81/5c505d508f7c887aa7982d21443a4126597c80d34b0bcf40f9cec576d7f3/grpcio-1.81.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a35009284d0d3d5c2c9601c164a911b8b4331608d98a9a66d47d97bb2f522b70", size = 8445238, upload-time = "2026-06-11T12:45:10.243Z" }, + { url = "https://files.pythonhosted.org/packages/f7/b2/524847365122ee509ca17bcc4e092198b700e94af7bfd5bb5e6dd9f3ee66/grpcio-1.81.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1b22c80559854b789a01fd89e8929b3798a156c0829b5282a8939f33ad4115ad", size = 7873989, upload-time = "2026-06-11T12:45:13.102Z" }, + { url = "https://files.pythonhosted.org/packages/18/fa/07c037c50b006909d1d13a5848774f8aa7b242f70dc03a035c64eea0e6db/grpcio-1.81.1-cp311-cp311-win32.whl", hash = "sha256:428bec0161b48d8cf583c068591bc0016d0d9cfff52462b72b3884861ea768c5", size = 4202223, upload-time = "2026-06-11T12:45:16.166Z" }, + { url = "https://files.pythonhosted.org/packages/41/ed/6bff15376920942fac6b95b9802752b837437172c9e8fc2d3170546b89cc/grpcio-1.81.1-cp311-cp311-win_amd64.whl", hash = "sha256:30e825f6848d9f18bba350ed6c75c1b02a0b5184474a31db9a32b1fa66fd8c79", size = 4941303, upload-time = "2026-06-11T12:45:18.724Z" }, { url = "https://files.pythonhosted.org/packages/85/07/9a979c81738863a738dc23d65177056e71fbb2db817740ed870b33434e7a/grpcio-1.81.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:8b39472beafc0bdcafc4c8c73ad082ebfdb449d566897a61e7acb4fa88089115", size = 6053264, upload-time = "2026-06-11T12:45:21.017Z" }, { url = "https://files.pythonhosted.org/packages/75/95/539706ca0d3bd40dbad583dc56fd883da941f37556b629132da5762781b9/grpcio-1.81.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:12b7524c88d4026d3dcb7b0ebe16b6714f3b4af402ddd0f0639ab064a00c87c3", size = 12052560, upload-time = "2026-06-11T12:45:23.652Z" }, { url = "https://files.pythonhosted.org/packages/e0/44/f257b7e0bd69c93b06c6cb8ac8d1b901ccb42bedabd83c1a4c77a71f8810/grpcio-1.81.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1e123f9b37edb8375fd74130d1f69c944bbf0a7b06761ae7211154b8759e94d2", size = 6595983, upload-time = "2026-06-11T12:45:26.963Z" }, @@ -648,6 +699,17 @@ version = "3.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, + { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, + { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, + { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, + { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, + { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, + { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, @@ -705,6 +767,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "nodeenv" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.43.0" @@ -765,69 +836,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/20/b685ed7af2e17c29ffc8af56f1fa8bc2033258fc30fb0d2b722f49d13ba0/opentelemetry_exporter_otlp_proto_http-1.43.0-py3-none-any.whl", hash = "sha256:647f603aa8efdbdb4dbff842e0729d0406a6fff26b295a72d3d60e7d963b2610", size = 21795, upload-time = "2026-06-24T15:19:43.164Z" }, ] -[[package]] -name = "opentelemetry-instrumentation" -version = "0.64b0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "packaging" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7e/97/02fe6e1c8b1ffac42d0b429c18080edb24e0e0d18c86612edf72b5752382/opentelemetry_instrumentation-0.64b0.tar.gz", hash = "sha256:b47d528dead6271d7743114417eb67fc915bd9258111c48dbf9a4951d2efa88d", size = 41935, upload-time = "2026-06-24T15:19:12.951Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/0c/cb9fe342de5299c7af24582eb7d788661cc53a1c4b904da92309caaa9417/opentelemetry_instrumentation-0.64b0-py3-none-any.whl", hash = "sha256:133ab7ffca796557aec059bf6be3190a34b6dea987f25be3d9409e230cbdad8b", size = 35880, upload-time = "2026-06-24T15:18:17.277Z" }, -] - -[[package]] -name = "opentelemetry-instrumentation-asgi" -version = "0.64b0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "asgiref" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-instrumentation" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "opentelemetry-util-http" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/85/0c/71c696fccb86d37af383ea1604af4729fabad0af2fabaf203e4c79c1e859/opentelemetry_instrumentation_asgi-0.64b0.tar.gz", hash = "sha256:4dd3eee566a4303f8e6b9b84f2a0a7abc57a6640df768926c68a3868bf5b2090", size = 26136, upload-time = "2026-06-24T15:19:17.003Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/20/218b65a63d847a7ed28d1bea84c39234689160b74480b8702272e37f4240/opentelemetry_instrumentation_asgi-0.64b0-py3-none-any.whl", hash = "sha256:e0840b66e15303a9254b0540946010bd008aa0504f4d89b8e1b7fb63490a36f0", size = 15906, upload-time = "2026-06-24T15:18:23.107Z" }, -] - -[[package]] -name = "opentelemetry-instrumentation-fastapi" -version = "0.64b0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-instrumentation" }, - { name = "opentelemetry-instrumentation-asgi" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "opentelemetry-util-http" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/db/a1/52282e2cc5c08f4df13b087896d9907258fe2ff4f34035d3b21aa92684c3/opentelemetry_instrumentation_fastapi-0.64b0.tar.gz", hash = "sha256:05f75149929e433c1630de381688e650bf651c1e1cce7f9a7b649a807dac8a98", size = 26236, upload-time = "2026-06-24T15:19:27.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/72/2d/4f48dc2d6f289f2febc5b871940dbea9f3b04ab9186d23342581b49cb984/opentelemetry_instrumentation_fastapi-0.64b0-py3-none-any.whl", hash = "sha256:43cbbfb2d3079dc81104478a2950ae93ac6d0e90a5020fa3987a236f8f2bdef1", size = 13263, upload-time = "2026-06-24T15:18:38.553Z" }, -] - -[[package]] -name = "opentelemetry-instrumentation-httpx" -version = "0.64b0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-instrumentation" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "opentelemetry-util-http" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0d/2a/2893a8781b93894f1e8014904c0342da7e4302de6597c2d5c0cb6c1a552e/opentelemetry_instrumentation_httpx-0.64b0.tar.gz", hash = "sha256:c2cfcd03d3665762860ebd0c28038c6e47fbb48d7942dec31dd75fc634d25c92", size = 23555, upload-time = "2026-06-24T15:19:29.107Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/29/a20309bd3f5a8051b61ca475e78623410c3b077e3deccae19aa4f8b5b9a2/opentelemetry_instrumentation_httpx-0.64b0-py3-none-any.whl", hash = "sha256:04829e5723941b5ceb0c88b44d63983e226b5c75b2b2e34a57739fdd0e060608", size = 16336, upload-time = "2026-06-24T15:18:41.412Z" }, -] - [[package]] name = "opentelemetry-proto" version = "1.43.0" @@ -867,15 +875,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/ca/23ba87a221b574a7c5a99d48849d80bfe8b047624681357e2b002e566187/opentelemetry_semantic_conventions-0.64b0-py3-none-any.whl", hash = "sha256:ea77e85e354b8f604ddbe5f3d9135216f982fa4d77e5859ac30f6d8a50505aa6", size = 203713, upload-time = "2026-06-24T15:19:53.339Z" }, ] -[[package]] -name = "opentelemetry-util-http" -version = "0.64b0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/1b/1029a805fd7242f7dfce91633b244c3b14a94d703232878f71e01ce862b1/opentelemetry_util_http-0.64b0.tar.gz", hash = "sha256:8a86a220dbfc56d736f47f1e5c4e7932a21fcf69052312e1bcf166444dc79322", size = 11102, upload-time = "2026-06-24T15:19:48.974Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1c/c7/5f8ec5b30546f2dc22cd5fc5759bce2ab5be6e89a2e710a405ac9ef64ed3/opentelemetry_util_http-0.64b0-py3-none-any.whl", hash = "sha256:c1e5350d25507c1afcd6076cf9ac062485a0a4f79cd9971366996fd3056bacdb", size = 8204, upload-time = "2026-06-24T15:19:09.02Z" }, -] - [[package]] name = "packaging" version = "26.2" @@ -898,25 +897,23 @@ wheels = [ name = "policyengine-observability" version = "1.4.1" source = { editable = "." } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-grpc" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, - { name = "opentelemetry-instrumentation-fastapi" }, - { name = "opentelemetry-instrumentation-httpx" }, - { name = "opentelemetry-sdk" }, -] [package.optional-dependencies] all = [ { name = "fastapi" }, { name = "flask" }, + { name = "google-auth" }, { name = "google-cloud-logging" }, { name = "httpx" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, ] dev = [ { name = "build" }, { name = "coverage" }, + { name = "pyright" }, { name = "pytest" }, { name = "ruff" }, { name = "towncrier" }, @@ -928,11 +925,26 @@ flask = [ { name = "flask" }, ] google = [ + { name = "google-auth" }, { name = "google-cloud-logging" }, ] httpx = [ { name = "httpx" }, ] +otel = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, +] +otlp-grpc = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-sdk" }, +] +otlp-http = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, +] [package.metadata] requires-dist = [ @@ -942,16 +954,25 @@ requires-dist = [ { name = "fastapi", marker = "extra == 'fastapi'" }, { name = "flask", marker = "extra == 'all'", specifier = ">=2.2" }, { name = "flask", marker = "extra == 'flask'", specifier = ">=2.2" }, + { name = "google-auth", marker = "extra == 'all'", specifier = ">=2.38.0" }, + { name = "google-auth", marker = "extra == 'google'", specifier = ">=2.38.0" }, { name = "google-cloud-logging", marker = "extra == 'all'", specifier = ">=3.15.0" }, { name = "google-cloud-logging", marker = "extra == 'google'", specifier = ">=3.15.0" }, { name = "httpx", marker = "extra == 'all'" }, { name = "httpx", marker = "extra == 'httpx'" }, - { name = "opentelemetry-api", specifier = ">=1.43.0" }, - { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.43.0" }, - { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.43.0" }, - { name = "opentelemetry-instrumentation-fastapi", specifier = ">=0.64b0" }, - { name = "opentelemetry-instrumentation-httpx", specifier = ">=0.64b0" }, - { name = "opentelemetry-sdk", specifier = ">=1.43.0" }, + { name = "opentelemetry-api", marker = "extra == 'all'", specifier = ">=1.43.0" }, + { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.43.0" }, + { name = "opentelemetry-api", marker = "extra == 'otlp-grpc'", specifier = ">=1.43.0" }, + { name = "opentelemetry-api", marker = "extra == 'otlp-http'", specifier = ">=1.43.0" }, + { name = "opentelemetry-exporter-otlp-proto-grpc", marker = "extra == 'all'", specifier = ">=1.43.0" }, + { name = "opentelemetry-exporter-otlp-proto-grpc", marker = "extra == 'otlp-grpc'", specifier = ">=1.43.0" }, + { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'all'", specifier = ">=1.43.0" }, + { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otlp-http'", specifier = ">=1.43.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'all'", specifier = ">=1.43.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.43.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'otlp-grpc'", specifier = ">=1.43.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'otlp-http'", specifier = ">=1.43.0" }, + { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.405" }, { name = "pytest", marker = "extra == 'dev'" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.9.0" }, { name = "towncrier", marker = "extra == 'dev'", specifier = ">=24.8.0" }, @@ -1039,6 +1060,21 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, @@ -1099,10 +1135,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, ] [[package]] @@ -1123,6 +1171,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bd/24/12818598c362d7f300f18e74db45963dbcb85150324092410c8b49405e42/pyproject_hooks-1.2.0-py3-none-any.whl", hash = "sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913", size = 10216, upload-time = "2024-09-29T09:24:11.978Z" }, ] +[[package]] +name = "pyright" +version = "1.1.414" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nodeenv" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/1b/244c7b710031ada80f27e579ec20d28a2285dfc318fed0339866b1047f12/pyright-1.1.414.tar.gz", hash = "sha256:523c0a97c60da6333234955c277730c9cf4f5bd6d5399e7b7d2b0fc5d3599524", size = 4154638, upload-time = "2026-09-10T12:26:53.181Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/ba/18b6e682ead424ad24bcc134339ae5d1b931cd9ae260540592a058a91279/pyright-1.1.414-py3-none-any.whl", hash = "sha256:2a6b4b3298c9eec174c5ed83bd338de6eee82df2992f3e1930e6199d381be36f", size = 6225049, upload-time = "2026-09-10T12:26:51.427Z" }, +] + [[package]] name = "pytest" version = "9.1.1" @@ -1246,67 +1307,3 @@ sdist = { url = "https://files.pythonhosted.org/packages/dd/b2/381be8cfdee792dd1 wheels = [ { url = "https://files.pythonhosted.org/packages/93/8c/2e650f2afeb7ee576912636c23ddb621c91ac6a98e66dc8d29c3c69446e1/werkzeug-3.1.8-py3-none-any.whl", hash = "sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50", size = 226459, upload-time = "2026-04-02T18:49:12.72Z" }, ] - -[[package]] -name = "wrapt" -version = "2.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fe/a4/282c8e64300a59fc834518a54bf0afabb4ff9218b5fa76958b450459a844/wrapt-2.2.2.tar.gz", hash = "sha256:0788e321027c999bf221b667bd4a54aaefd1a36283749a860ac3eb77daed0302", size = 129068, upload-time = "2026-06-20T23:49:44.49Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/85/180b40628b23772692a0c76e8030114e1c0ae068470ed531919f0a5f2a4a/wrapt-2.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8417fd3c674d3c8023d080292d29301531a12daf8bd938dd419710dd2f464f2b", size = 81484, upload-time = "2026-06-20T23:47:59.924Z" }, - { url = "https://files.pythonhosted.org/packages/94/f2/21c90f2a16689702e2aaff45795b11018dff2c9b1242bac10d225483f676/wrapt-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e7070c7472582e31af3dfc2622b2381a0df7435110a9388ed8db5ffbce67efb", size = 82151, upload-time = "2026-06-20T23:48:01.303Z" }, - { url = "https://files.pythonhosted.org/packages/5f/b3/7e6e9fcf4fe7e1b69a49fe6cc5a44e8224bab6283c5233c97e132f14908e/wrapt-2.2.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e096c9d39a59b35b63c9aacfbbbec2088ff51ff1fc31051acc60a07f42f273a", size = 169828, upload-time = "2026-06-20T23:48:02.719Z" }, - { url = "https://files.pythonhosted.org/packages/0b/43/894f132d857ed5a9904d937baf368badcbe5ea9e436e2f1930fe21c9f1f0/wrapt-2.2.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d1a6050405bf334be33bf66296f113563622972a34900ae6fa60fd283a1a900", size = 171544, upload-time = "2026-06-20T23:48:04.266Z" }, - { url = "https://files.pythonhosted.org/packages/29/de/3c833e03725b477e9ea34028224dd21a48781830101e4e036f77e8b6b102/wrapt-2.2.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10adb01371408c6de504a6658b9886480f1a4919a83752748a387a504a21df79", size = 160663, upload-time = "2026-06-20T23:48:05.708Z" }, - { url = "https://files.pythonhosted.org/packages/33/be/27edce350b24e3054d9d047f65f16d4c4d4c1f3f31c4278a1f8a95c723c8/wrapt-2.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3442eee2a5798f9b451f1b2cd7518ce8b7e28a2a364696c414460a0e295c012a", size = 169387, upload-time = "2026-06-20T23:48:07.243Z" }, - { url = "https://files.pythonhosted.org/packages/e2/c4/9fd9679af8bf38e146652c7f47b6b352c3e5795b4ad1c0b7f94e15ac2aa7/wrapt-2.2.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6c99012a22f735a85eed7c4b86a3e99c30fdd57d9e115b2b45f796264b58d0bf", size = 158849, upload-time = "2026-06-20T23:48:08.91Z" }, - { url = "https://files.pythonhosted.org/packages/bc/c2/aa6c0c2206803068c6859dabe01f8c84c43744da93d4c67b8946d21655ee/wrapt-2.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b686cfc008776a3952d6213cb296ed7f45d782a8453936406faa89eac0835ab", size = 168147, upload-time = "2026-06-20T23:48:10.374Z" }, - { url = "https://files.pythonhosted.org/packages/42/63/3eb25da41049d20ae18fcab2dd8b056e02387c4bfa626cbdfb7c3b872e4f/wrapt-2.2.2-cp312-cp312-win32.whl", hash = "sha256:ef2cce266b5b0b07e19fa82e59673b81142b7a3607c8ed1254113d048ed668da", size = 77734, upload-time = "2026-06-20T23:48:11.769Z" }, - { url = "https://files.pythonhosted.org/packages/da/09/0390e008a305360948fa9ce69507d041ac12cb2ee5d28e34467e2ee79391/wrapt-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:abf8c20a2d72ee69e16328b3c91342c446e723bfe48bfcc4dded3b9722ac027f", size = 80585, upload-time = "2026-06-20T23:48:13.117Z" }, - { url = "https://files.pythonhosted.org/packages/d3/b3/84c445c66969f2d3457276b183a48c91097d59bbef9af6c075366b0f8c36/wrapt-2.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:c6c64c5d02578bc4c4bca4f0aef1504de933c1d5b4ac2710b9131111459506c8", size = 79553, upload-time = "2026-06-20T23:48:14.5Z" }, - { url = "https://files.pythonhosted.org/packages/43/fc/f32f4b22c6511173c11d9e541ab4e7d8467a0f1b3455acaf784115d31ff8/wrapt-2.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e8b648270c613720a202d9a45ebabc33261b22c3a839b115ac5bce8c0bb0d69", size = 81296, upload-time = "2026-06-20T23:48:15.881Z" }, - { url = "https://files.pythonhosted.org/packages/72/06/4d117d5d77a9344776c0248b24dae3d3dd2f58e5f765fa08cf887072e719/wrapt-2.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6fb7e94e8fe3e4c3067bb1653a91cce7c5e83acc119fdd41501b1bf74654617", size = 81841, upload-time = "2026-06-20T23:48:17.262Z" }, - { url = "https://files.pythonhosted.org/packages/15/ff/63ad96f98eb58a742b1a20d80f21da88924405910149950b912368150468/wrapt-2.2.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb18fc51e813df0d9c98049e3bf2298a5495a648602040e21fa3c7329371159e", size = 167882, upload-time = "2026-06-20T23:48:18.764Z" }, - { url = "https://files.pythonhosted.org/packages/20/1f/8bb62d8933df7acf3247194e6e9fc68edf9d2fa203252c89c94b319dd472/wrapt-2.2.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94b00b00f806eb3ef2abe9049ed45994a81ee9284884d96e6b8314927c6cea3d", size = 167411, upload-time = "2026-06-20T23:48:20.315Z" }, - { url = "https://files.pythonhosted.org/packages/17/09/8789dcb09ee1de715727db7521aabbb68ffa68dfade3a49468440cfced49/wrapt-2.2.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:62415fd095bc590b842b6d092f2b5d9ccbaeb7e0b28535c03dcea2718b48636b", size = 158607, upload-time = "2026-06-20T23:48:21.728Z" }, - { url = "https://files.pythonhosted.org/packages/9c/20/66e02562d53ee67d841f175e38e3c993c2d78a3e104c576cad61c028b43c/wrapt-2.2.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a41e758d80dc0ab8c210f641ac892009d356cf1f955d97db544c8dd317b4d14c", size = 166367, upload-time = "2026-06-20T23:48:23.177Z" }, - { url = "https://files.pythonhosted.org/packages/bd/a3/832ac4e41222fb263b3042d42c2f08d305db7d0f0c9b1d3a271a9eede8f6/wrapt-2.2.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:b84cd4058001c9727b0e9980b7a9e66325b5ca748b1b578e822cade1bc6b304f", size = 157176, upload-time = "2026-06-20T23:48:24.711Z" }, - { url = "https://files.pythonhosted.org/packages/b7/01/1bd5e4d2df9c0178989ac8da9186543465388588ee2ef153e2591accebef/wrapt-2.2.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:26fc73a1b15e0946d2942b9a4426d162b51676338327dc067ccd8d2d76385f94", size = 167025, upload-time = "2026-06-20T23:48:26.118Z" }, - { url = "https://files.pythonhosted.org/packages/1c/69/583ed25291ab53e1ec117135fb1c33425e2f46d2bc8f29c17f7a94cf4274/wrapt-2.2.2-cp313-cp313-win32.whl", hash = "sha256:3c4095803491f6ef72128914c28ec05bbad9758433bb35f6715a3e9c8e46fb2d", size = 77605, upload-time = "2026-06-20T23:48:27.643Z" }, - { url = "https://files.pythonhosted.org/packages/29/68/e69fc6d06e1523c68e0d00f95c9aed1158ce9908ee41603f7f2eae3d5db6/wrapt-2.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:2cb07f414fab25dbe6b5c7398e1491423a5c81a6209533639969a6c928d474a4", size = 80508, upload-time = "2026-06-20T23:48:29.013Z" }, - { url = "https://files.pythonhosted.org/packages/55/21/fe7a393d9e5dc0923bed8f5d857e9dcff210f1fa0888c02cc8f3ffaa55aa/wrapt-2.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:1fc7691f070220215cccb2a20836b9adbaecb8ff22ad47abe63de5f110994fac", size = 79565, upload-time = "2026-06-20T23:48:30.429Z" }, - { url = "https://files.pythonhosted.org/packages/b6/e5/c120d13bf5091164f68c3c1657e84f16f57e71d978421b626393ac5bd7eb/wrapt-2.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ec8f83949028366531383603139403cac7a826e4011955813cdd640017845ce5", size = 83264, upload-time = "2026-06-20T23:48:31.807Z" }, - { url = "https://files.pythonhosted.org/packages/d3/b0/d4a1eb97e0e286625bdf21bc7f702637f9607787ffbbdb5ec14d50c79dbf/wrapt-2.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4b481fb0c40d9fd90a5809911208da700987d373a20a4709dc9e3944af7a6bec", size = 83791, upload-time = "2026-06-20T23:48:33.482Z" }, - { url = "https://files.pythonhosted.org/packages/18/1e/f060df47755e87b57684cee7bfc1362b204df55fac96ffebc0631b697b79/wrapt-2.2.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0065a3b657cec06813b4241d2462ccec287f6863103d7445b725fb3a889736f9", size = 203399, upload-time = "2026-06-20T23:48:34.97Z" }, - { url = "https://files.pythonhosted.org/packages/c4/de/2316a757a1abb6453700b79d83e532146dcef2611348282d4d8889792161/wrapt-2.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30f7424af5c5c345b7f26490e097f74a2ef45b3d08b664dc33571aee3bd3b56c", size = 210461, upload-time = "2026-06-20T23:48:36.569Z" }, - { url = "https://files.pythonhosted.org/packages/ed/29/d1160785ae18ca2495a6d82a21154103d74f656c9fd457fb35f6b11b965a/wrapt-2.2.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:07fdcb012821859168641acf68afad61ef9783cf37100af85f152550e9677194", size = 195313, upload-time = "2026-06-20T23:48:38.175Z" }, - { url = "https://files.pythonhosted.org/packages/f5/2d/7caa9598ae61a9cf0989cc501739cbeeb7d650ab3193cca1407b9af0c6ab/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f90038ab58fafb584801ca62d72384d7d5225d93c76f7b773c22fae545bd8066", size = 206116, upload-time = "2026-06-20T23:48:39.804Z" }, - { url = "https://files.pythonhosted.org/packages/ac/02/281ea1088b8650d865f311b35cf86fd21df89128e2909714f1161e01c9d0/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:c5d7825491bfa2d08b97e9557768987952c7b9ae687d06c3320b40a37ccb7f20", size = 192668, upload-time = "2026-06-20T23:48:41.346Z" }, - { url = "https://files.pythonhosted.org/packages/be/7d/976e2d5b4b5c5babda40974edd54d0a5585cb60132ed86b46f4b80239b16/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ad520e6daa9bbf136f14de735474dbec7dcc0891f718e1d274ce8dc92e645af", size = 198891, upload-time = "2026-06-20T23:48:43.056Z" }, - { url = "https://files.pythonhosted.org/packages/59/b7/e47651797c097f75a37e2ce86dcf04048ff576f3a674f7c558df7b5e9622/wrapt-2.2.2-cp313-cp313t-win32.whl", hash = "sha256:25904acb9475f46c24fe0423dbc8fda8cc5fbc282ab3dc6e72e919748c53f4e9", size = 78537, upload-time = "2026-06-20T23:48:44.509Z" }, - { url = "https://files.pythonhosted.org/packages/d1/6f/9fa5d59fb06d890defb5a8f727ce6a14d2932c8760153f96956628559fee/wrapt-2.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:305d4c247d61c4115794a169141823c62f719525ddb90b23aa332741c77d2c28", size = 82005, upload-time = "2026-06-20T23:48:46.391Z" }, - { url = "https://files.pythonhosted.org/packages/15/80/4c7bd9873d1f9f7d138d93556b500469dbe24f42710b877519c2b9eb380d/wrapt-2.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:c20279cd1a29800815d7b2d6338b60a6c6e78263f9d6e62e0eda251ba9cae2d0", size = 80762, upload-time = "2026-06-20T23:48:47.964Z" }, - { url = "https://files.pythonhosted.org/packages/24/05/7fd9c3f83b2c74cbfc572a0b88aa37431e04bd8aed70d2c0efd3464206de/wrapt-2.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0e64826f920c42d9d9f87e8cc09ffae66c51ede12d59061a5a426deb9aa71745", size = 81341, upload-time = "2026-06-20T23:48:49.39Z" }, - { url = "https://files.pythonhosted.org/packages/4b/68/1bfa43100dd90d4ef74a05897b86275cf57e1313ca14aae2545bc9f872c9/wrapt-2.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dcaa5e1451bd8751d7bd1568dfa3321c78092a52a7ecb5d1a0f18a5791e1fd00", size = 81921, upload-time = "2026-06-20T23:48:50.986Z" }, - { url = "https://files.pythonhosted.org/packages/74/eb/df7b7f0b631dbbc750f39be27d8b55f65777d8ac86da80e12be41a644c4b/wrapt-2.2.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0abfd648dac9ac9c5b3aa9b523d27f1789046640b58dcd5652a720ddb325e1fc", size = 167713, upload-time = "2026-06-20T23:48:52.598Z" }, - { url = "https://files.pythonhosted.org/packages/4d/9a/d1bd36f6d088c8e652a9383cabbd49af30b8c576302a7eccddbab6963e3f/wrapt-2.2.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f4bfd8d1eb438153eff8b8cfe87f032ba65731e1ce06138b5090f745a33f6f95", size = 166779, upload-time = "2026-06-20T23:48:54.33Z" }, - { url = "https://files.pythonhosted.org/packages/4c/ae/24ffacd4187fac2740a1972093929e836dea092d42c87d728cd98fee11a6/wrapt-2.2.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c427c9d06d859848a69f0d928fe28b5c33a941b2265d10a0e1f15cd244f1ee33", size = 158407, upload-time = "2026-06-20T23:48:55.944Z" }, - { url = "https://files.pythonhosted.org/packages/a3/ed/974427668249a356051e8d67d47fa54ef6c777f0fcf3bae9d292c047d4b6/wrapt-2.2.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4250b43d1a129d947e083c4dc6baf333c9bb34edd26f912d5b0457841fc858ab", size = 166594, upload-time = "2026-06-20T23:48:57.617Z" }, - { url = "https://files.pythonhosted.org/packages/fb/5f/e1d7c6e4523f78db2fbd7826babd0348da1d5e0834c4f918b9ab5757dfae/wrapt-2.2.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:173e5bb5ca350a6e0abab60b7ec7cdd7992a814cb14b4de670a28f067f105663", size = 157068, upload-time = "2026-06-20T23:48:59.171Z" }, - { url = "https://files.pythonhosted.org/packages/1e/c1/7ebd1027f00700c0b0233b20aceef2b4784294ed64971424c4a78e069e34/wrapt-2.2.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aa14b01804bce36c6d63d7b6a4f55df390f29f8648cc13a1f40b166f4d54680d", size = 166470, upload-time = "2026-06-20T23:49:00.737Z" }, - { url = "https://files.pythonhosted.org/packages/99/eb/974e471a6a978b8180186b8a9dc5ae3361ce269a967190b709b8ce17abfb/wrapt-2.2.2-cp314-cp314-win32.whl", hash = "sha256:58f9f8d637c9a6e245c6ef5b109b67ec187d2faed23d1405656b51d96e0a5b56", size = 78062, upload-time = "2026-06-20T23:49:02.327Z" }, - { url = "https://files.pythonhosted.org/packages/49/ec/e1281156cdc7a66693838ad7a0865ad641c74abd337a957d668b575aaffb/wrapt-2.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:385cb1866f20479e83299af585375bfa0a4b0c6c9907a981483ea782ea8ae406", size = 80832, upload-time = "2026-06-20T23:49:03.837Z" }, - { url = "https://files.pythonhosted.org/packages/45/7d/1b6b5ddd94005a2dac97a4490c9838f3154977850d633abcb65b30089437/wrapt-2.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:8ffbeaea6771a6eba6e6eeb09767864995726bc8240bb54baf88a9bb1db34d5c", size = 80029, upload-time = "2026-06-20T23:49:05.237Z" }, - { url = "https://files.pythonhosted.org/packages/b0/33/9ebcf8aafe91c601127cbd93708c16aa8f688f34a10bf004046803ecdc4f/wrapt-2.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:09f811d43f6f33ec7515f0be76b159569f4057ab54d3e079c3204dddb90afa2a", size = 83357, upload-time = "2026-06-20T23:49:06.632Z" }, - { url = "https://files.pythonhosted.org/packages/39/38/ec45b635153327b52e52732a0ea980e5f00b7efba65f9e018828f1e69daa/wrapt-2.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a795d3c06e5fbf9ea2f13196180b77aeab1b4685917256ee0d014cc163d90063", size = 83794, upload-time = "2026-06-20T23:49:08.098Z" }, - { url = "https://files.pythonhosted.org/packages/4e/ea/1a89e6d3b7a83c3affe5c09cde77792c947e63e4bc85ad84cd5bb9abb0d8/wrapt-2.2.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:45c2f2768e790c9f8db90f239ef23a2af8e7570f25a35619ef902df4a738447f", size = 203362, upload-time = "2026-06-20T23:49:09.811Z" }, - { url = "https://files.pythonhosted.org/packages/19/d8/3b58763d9863b5a73771c0d97110f9595d248db454009e07e1535ee905a4/wrapt-2.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbf00ee0cb55ec24e2b0995a71942b85b21a066db8f3f46e1dbfdb9433ffba81", size = 210449, upload-time = "2026-06-20T23:49:11.521Z" }, - { url = "https://files.pythonhosted.org/packages/2d/6f/17fd9e053103d8be148d20d5d7505facc72d5fe1f9127973904ceaed79cf/wrapt-2.2.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2252f77663651b89255895f58cc6ac08fcb206d4371813e5af61bb62d4f7689c", size = 195349, upload-time = "2026-06-20T23:49:13.346Z" }, - { url = "https://files.pythonhosted.org/packages/ef/04/d0d1ccaaa12cb7dccf28a23f0279a608ba498f71e81d949d5ed54bcfd5c1/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cd7181ab1c31192ff5219269830744b5a62020b3a6d433588c4f1c95b8f8bff", size = 206099, upload-time = "2026-06-20T23:49:15.051Z" }, - { url = "https://files.pythonhosted.org/packages/44/b3/e8aa07b619890a2aa6cde1931b1887abb08820721b564a5f80b7ca3f3aa0/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:6fe35fd51b74867d8b80174c277bd6bbf6a73e443f908129dc531c4b688a20d5", size = 192728, upload-time = "2026-06-20T23:49:16.854Z" }, - { url = "https://files.pythonhosted.org/packages/b7/f0/1819fb50f0d3c9bd758d8a83b56f1b470dee8b5b8eac8702b7c137cea9d4/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:11d95fc2fbad3163596c39d440e6f21ca9fccece74b56e30a37ac2fca786a07c", size = 198842, upload-time = "2026-06-20T23:49:18.504Z" }, - { url = "https://files.pythonhosted.org/packages/67/7c/e88313f16a99930b899ef970d91c281544a470749a359decad994483bbda/wrapt-2.2.2-cp314-cp314t-win32.whl", hash = "sha256:d8a15813215f33fa83667bfc978b300e35669ea8bb424e970a1426bcb7bc6cca", size = 79059, upload-time = "2026-06-20T23:49:20.107Z" }, - { url = "https://files.pythonhosted.org/packages/a0/4f/ac12fda57a55068a094ec42851fb0a40e8489d8941863d517452de62e507/wrapt-2.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d09db0f7e8357060d3c38fc22a018aba683a796bf184360fd1a58f6fc180dc77", size = 82462, upload-time = "2026-06-20T23:49:21.631Z" }, - { url = "https://files.pythonhosted.org/packages/48/a7/df732dac86d9b2027c56bd163dbc883e037b16c3469614752e148d219c61/wrapt-2.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:f32fe639c39561ccc187bcae17e9271be0eb45f1c2952510d2f29b33ab577347", size = 81182, upload-time = "2026-06-20T23:49:23.199Z" }, - { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" }, -] From c0537055990883cfd1adc349463c6d29ba3407d2 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 22 Sep 2026 03:57:45 +0400 Subject: [PATCH 02/12] Restrict central log writers --- deploy/gcp/README.md | 33 +++++++++++++++++++++++++++++++++ deploy/gcp/verify.sh | 12 ++++++++++++ 2 files changed, 45 insertions(+) diff --git a/deploy/gcp/README.md b/deploy/gcp/README.md index a720485..c2eadd6 100644 --- a/deploy/gcp/README.md +++ b/deploy/gcp/README.md @@ -66,6 +66,12 @@ Grant only the roles listed in [`iam.yaml`](iam.yaml). The collector receives existing Cloud Run identities in the inventory receive collector invocation permission on the collector service only. +Remove project-level `roles/logging.logWriter` bindings from every identity +outside this inventory. Source-project logging service agents use conditional +`roles/logging.bucketWriter` access to the named analytics bucket and do not +receive project-level log write access. `verify.sh` fails when another +project-level log writer is present. + Create a separate `modal-api-v1` workload identity pool and provider using the issuer, audience, mappings, workspace, environment, and application condition in `iam.yaml`. Do not modify the existing `modal/modal` provider during this @@ -179,3 +185,30 @@ verify the environment restriction, then delete that environment. Rollback does not modify the existing `modal/modal` provider or any excluded application deployment. + +## Deployment record + +The infrastructure portion of this runbook was applied and verified on +2026-09-22: + +- the global `policyengine-observability` log bucket retains records for 30 + days and has log analytics enabled; +- exact source-project sinks route the two API services and the two simulation + entry services to that bucket; +- the authenticated collector runs in `us-central1` as + `policyengine-api-v1-otel-collector`; +- the dedicated `modal-api-v1` identity provider is active with the workspace, + environment, and application conditions in `iam.yaml`; +- the only project-level `roles/logging.logWriter` identity is + `policyengine-api-v1-modal`; +- the dashboard and six alert policies are present and enabled; and +- the project currently has no alert notification channel, so the policies + record incidents without sending email, Slack, or paging notifications. + +The package and consumer service rollout remains pending until the three draft +pull requests are reviewed, the package is published as version 2.0, and the +temporary Git source pins in both consumer repositories are replaced with the +published version. Run the synthetic cross-service request, volume and cost +measurement, and destination comparison after those deployments. Record the +deployed revisions and the observation interval here before declaring the +consumer rollout complete. diff --git a/deploy/gcp/verify.sh b/deploy/gcp/verify.sh index e454dfc..eaed4c5 100644 --- a/deploy/gcp/verify.sh +++ b/deploy/gcp/verify.sh @@ -20,6 +20,18 @@ gcloud projects get-iam-policy "${project}" \ --filter='bindings.role:roles/telemetry.writer OR bindings.role:roles/serviceusage.serviceUsageConsumer OR bindings.role:roles/logging.logWriter' \ --format='table(bindings.role,bindings.members)' +log_writers="$( + gcloud projects get-iam-policy "${project}" \ + --flatten='bindings[].members' \ + --filter='bindings.role=roles/logging.logWriter' \ + --format='value(bindings.members)' +)" +expected_log_writer='serviceAccount:policyengine-api-v1-modal@policyengine-observability.iam.gserviceaccount.com' +if [[ "${log_writers}" != "${expected_log_writer}" ]]; then + echo "Unexpected project-level Cloud Logging writers: ${log_writers}" >&2 + exit 1 +fi + gcloud logging views get-iam-policy _AllLogs \ --bucket=policyengine-observability \ --location=global \ From 92eb8237c0dc23678d1837ba21f02b91cf557240 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 22 Sep 2026 16:13:39 +0400 Subject: [PATCH 03/12] Contain observability lifecycle failures --- README.md | 20 +- changelog.d/29.breaking.md | 4 +- .../adapters/fastapi.py | 19 +- policyengine_observability/config.py | 4 +- policyengine_observability/delivery.py | 16 +- policyengine_observability/diagnostics.py | 7 + policyengine_observability/runtime.py | 73 +++++-- tests/test_adapters.py | 27 +++ tests/test_delivery.py | 7 +- tests/test_runtime.py | 179 ++++++++++++++++++ 10 files changed, 314 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 455b945..003b2f8 100644 --- a/README.md +++ b/README.md @@ -31,12 +31,14 @@ environment variables may supply OTLP transport settings. ```python from policyengine_observability import ( DeploymentIdentity, + LoggingConfig, ObservabilityConfig, + OTelConfig, ServiceIdentity, configure, ) -config = ObservabilityConfig.from_env( +config = ObservabilityConfig( service=ServiceIdentity( name="policyengine-api", namespace="policyengine.api-v1", @@ -49,6 +51,12 @@ config = ObservabilityConfig.from_env( region="us-central1", ), google_cloud_project_id="policyengine-observability", + logging=LoggingConfig(shutdown_timeout_seconds=2), + otel=OTelConfig( + endpoint="https://COLLECTOR_HOST", + google_audience="https://COLLECTOR_HOST", + shutdown_timeout_seconds=3, + ), ) runtime = configure(config) ``` @@ -224,8 +232,8 @@ Remote logging starts no client until the background worker receives its first record. The calling thread uses `put_nowait`; a full queue drops the newest record and records a local counter. -After a Modal memory snapshot restores, rebuild process-local queues, threads, -credentials, and exporters before accepting work: +After a Modal memory snapshot restores, rebuild process-local locks, context, +queues, threads, credentials, and exporters before accepting work: ```python @modal.enter(snap=False) @@ -241,8 +249,10 @@ invalid configuration, unavailable DNS, denied permissions, collector failure, queue saturation, and shutdown timeouts produce rate-limited JSON diagnostics on standard error. Those diagnostics do not enter the failing exporter. -Call `runtime.shutdown()` during orderly process shutdown. The call is bounded -and safe to repeat. +Call `runtime.shutdown()` during orderly process shutdown. Logging and OTel +each use the timeout configured on their respective configuration objects. A +failure in either subsystem is reported locally and does not prevent cleanup +of the other subsystem. The call is safe to repeat. The operating policy, workload allowlist, Google Cloud deployment plan, and rollback procedure are in diff --git a/changelog.d/29.breaking.md b/changelog.d/29.breaking.md index 11ae2ab..1c70f05 100644 --- a/changelog.d/29.breaking.md +++ b/changelog.d/29.breaking.md @@ -3,4 +3,6 @@ runtime returned by `configure`. Version 2 adds runtime-bound operations and spans, idempotent Flask and FastAPI request adapters, explicit HTTPX and Python logging instrumentation, bounded Modal Cloud Logging delivery, authenticated OTLP export, and allowlisted cross-service correlation. Consumers must replace -removed version 1 imports before upgrading. +removed version 1 imports before upgrading. Logging and OTel now have separate +shutdown timeout settings, shutdown failures are contained per subsystem, and +process recovery replaces inherited locks and asynchronous delivery state. diff --git a/policyengine_observability/adapters/fastapi.py b/policyengine_observability/adapters/fastapi.py index c5817ee..7cb18da 100644 --- a/policyengine_observability/adapters/fastapi.py +++ b/policyengine_observability/adapters/fastapi.py @@ -56,13 +56,18 @@ async def send_observed(message: dict[str, Any]) -> None: if message.get("type") == "http.response.start": status_code = int(message.get("status") or 0) self._update_route(scope) - message = { - **message, - "headers": _merge_headers( - list(message.get("headers") or []), - self.runtime.response_headers(), - ), - } + try: + message = { + **message, + "headers": _merge_headers( + list(message.get("headers") or []), + self.runtime.response_headers(), + ), + } + except Exception as exc: + self.runtime.diagnostics.report( + "fastapi.response_headers", exc + ) await send(message) if message.get("type") == "http.response.body" and not message.get( "more_body", False diff --git a/policyengine_observability/config.py b/policyengine_observability/config.py index 6a6afa3..f64aa1e 100644 --- a/policyengine_observability/config.py +++ b/policyengine_observability/config.py @@ -68,7 +68,6 @@ class GoogleCloudLoggingConfig: queue_capacity: int = 1_000 batch_size: int = 1 write_timeout_seconds: float = 5.0 - close_timeout_seconds: float = 2.0 @dataclass(frozen=True, slots=True) @@ -78,6 +77,7 @@ class LoggingConfig: capture_standard_library: bool = False replace_existing_handlers: bool = False minimum_severity: int = 20 + shutdown_timeout_seconds: float = 2.0 @dataclass(frozen=True, slots=True) @@ -96,6 +96,7 @@ class OTelConfig: span_schedule_delay_seconds: float = 5.0 export_timeout_seconds: float = 5.0 metric_export_interval_seconds: float = 60.0 + shutdown_timeout_seconds: float = 3.0 @dataclass(frozen=True, slots=True) @@ -104,7 +105,6 @@ class TelemetryLimits: max_string_length: int = 1_024 max_error_message_length: int = 2_048 max_stack_length: int = 16_384 - shutdown_timeout_seconds: float = 3.0 async_parent_max_age_seconds: float = 300.0 diff --git a/policyengine_observability/delivery.py b/policyengine_observability/delivery.py index 4053a05..0805ddd 100644 --- a/policyengine_observability/delivery.py +++ b/policyengine_observability/delivery.py @@ -70,7 +70,12 @@ def emit(self, record: dict[str, Any]) -> None: def close(self, timeout_seconds: float | None = None) -> None: if self._remote is None: return - self._remote.close(timeout_seconds) + timeout = ( + self.config.logging.shutdown_timeout_seconds + if timeout_seconds is None + else timeout_seconds + ) + self._remote.close(timeout) @property def remote_enabled(self) -> bool: @@ -124,7 +129,7 @@ def enqueue(self, record: dict[str, Any]) -> None: def queue_depth(self) -> int: return self._queue.qsize() - def close(self, timeout_seconds: float | None = None) -> None: + def close(self, timeout_seconds: float) -> None: if self._closed.is_set(): return self._closed.set() @@ -132,12 +137,7 @@ def close(self, timeout_seconds: float | None = None) -> None: self._queue.put_nowait(self._STOP) except queue.Full: pass - timeout = ( - self.config.close_timeout_seconds - if timeout_seconds is None - else timeout_seconds - ) - self._thread.join(max(0.0, timeout)) + self._thread.join(max(0.0, timeout_seconds)) if self._thread.is_alive(): self.diagnostics.increment("logs.shutdown_timeout") self.diagnostics.report( diff --git a/policyengine_observability/diagnostics.py b/policyengine_observability/diagnostics.py index eb6e84f..838723d 100644 --- a/policyengine_observability/diagnostics.py +++ b/policyengine_observability/diagnostics.py @@ -48,6 +48,13 @@ def snapshot(self) -> dict[str, int]: with self._lock: return dict(self._counts) + def restart_after_process_duplication(self) -> None: + """Replace synchronization state inherited across process copying.""" + + self._lock = threading.Lock() + self._last_report = {} + self._counts = Counter() + def report( self, operation: str, diff --git a/policyengine_observability/runtime.py b/policyengine_observability/runtime.py index a37481c..8113286 100644 --- a/policyengine_observability/runtime.py +++ b/policyengine_observability/runtime.py @@ -2,6 +2,7 @@ import inspect import logging +import math import re import threading import time @@ -427,28 +428,56 @@ def shutdown(self) -> None: if self._closed: return self._closed = True - timeout = max(0.0, self.config.limits.shutdown_timeout_seconds) - self._delivery.close(timeout) - _run_bounded( - lambda: self._otel.shutdown(timeout), - timeout, - self.diagnostics, - "otel.shutdown_deadline", + logging_timeout = _bounded_shutdown_timeout( + self.config.logging.shutdown_timeout_seconds, + default=2.0, + ) + otel_timeout = _bounded_shutdown_timeout( + self.config.otel.shutdown_timeout_seconds, + default=3.0, ) - self._remove_logging_handlers() + try: + try: + self._delivery.close(logging_timeout) + except Exception as exc: + self.diagnostics.report("logging.shutdown", exc) + try: + _run_bounded( + lambda: self._otel.shutdown(otel_timeout), + otel_timeout, + self.diagnostics, + "otel.shutdown_deadline", + ) + except Exception as exc: + self.diagnostics.report("otel.shutdown", exc) + finally: + self._remove_logging_handlers() def restart_after_snapshot(self) -> None: - with self._shutdown_lock: - timeout = min( - max(0.0, self.config.limits.shutdown_timeout_seconds), 1.0 - ) + """Rebuild process-local state after a fork or snapshot restore. + + Call this only in a single-threaded lifecycle callback before the + copied process accepts application work. Inherited workers and locks + are abandoned because their owning threads may not exist in the new + process. + """ + + self._shutdown_lock = threading.Lock() + self.diagnostics.restart_after_process_duplication() + self._request_state = ContextVar( + f"policyengine_request_{id(self)}", default=None + ) + self._operation_state = ContextVar( + f"policyengine_operation_{id(self)}", default=None + ) + for _logger, handler in self._logging_handlers: try: - self._delivery.close(timeout) + handler.createLock() except Exception as exc: - self.diagnostics.report("snapshot.delivery_close", exc) - self._delivery = self._new_delivery() - self._otel = self._new_otel() - self._closed = False + self.diagnostics.report("process.logging_handler_lock", exc) + self._delivery = self._new_delivery() + self._otel = self._new_otel() + self._closed = False def _start_operation( self, @@ -855,3 +884,13 @@ def run() -> None: diagnostic_name, "Operation exceeded the configured shutdown deadline.", ) + + +def _bounded_shutdown_timeout(value: float, *, default: float) -> float: + try: + parsed = float(value) + except (TypeError, ValueError): + return default + if not math.isfinite(parsed): + return default + return min(max(parsed, 0.0), 60.0) diff --git a/tests/test_adapters.py b/tests/test_adapters.py index d99a3e2..9b020ca 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -80,6 +80,33 @@ async def item(item_id: str): unused.shutdown() +def test_fastapi_response_observability_failure_preserves_response( + monkeypatch, +) -> None: + runtime, output = make_runtime() + app = FastAPI() + + @app.get("/ok") + async def ok(): + return {"status": "ok"} + + def fail_response_headers(): + raise RuntimeError("response instrumentation failed") + + monkeypatch.setattr(runtime, "response_headers", fail_response_headers) + instrument_fastapi(app, runtime) + + with TestClient(app, raise_server_exceptions=False) as client: + response = client.get("/ok") + + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + assert REQUEST_ID_HEADER not in response.headers + assert runtime.diagnostics.count("failure.fastapi.response_headers") == 1 + assert records(output)[0]["outcome"] == "success" + runtime.shutdown() + + def test_fastapi_error_and_non_http_scope() -> None: runtime, output = make_runtime() app = FastAPI() diff --git a/tests/test_delivery.py b/tests/test_delivery.py index 294ce06..0ab15cf 100644 --- a/tests/test_delivery.py +++ b/tests/test_delivery.py @@ -42,14 +42,17 @@ def modal_config(**remote_overrides): "project_id": "policyengine-observability", "queue_capacity": 10, "batch_size": 5, - "close_timeout_seconds": 0.2, } + shutdown_timeout_seconds = remote_overrides.pop( + "shutdown_timeout_seconds", 0.2 + ) remote_values.update(remote_overrides) return make_config( deployment=DeploymentIdentity("test", "modal"), logging=LoggingConfig( stdout_enabled=True, remote=GoogleCloudLoggingConfig(**remote_values), + shutdown_timeout_seconds=shutdown_timeout_seconds, ), ) @@ -197,7 +200,7 @@ def write_many(self, records: list[dict]) -> None: release.wait(1) manager = DeliveryManager( - modal_config(close_timeout_seconds=0.01), + modal_config(shutdown_timeout_seconds=0.01), diagnostics, stdout=io.StringIO(), writer_factory=lambda _config: BlockingWriter(), diff --git a/tests/test_runtime.py b/tests/test_runtime.py index b61a913..379f74f 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -2,12 +2,19 @@ import asyncio import logging +import os +import select +import signal +import threading +import time import pytest from conftest import make_config, make_runtime, records from policyengine_observability import ( REQUEST_ID_HEADER, + LoggingConfig, + OTelConfig, configure, instrument_logging, ) @@ -266,6 +273,178 @@ def test_shutdown_is_bounded_repeatable_and_restartable(runtime) -> None: assert not observed._closed +def test_shutdown_contains_delivery_failure_and_continues_cleanup( + monkeypatch, +) -> None: + observed, _output = make_runtime( + logging=LoggingConfig(shutdown_timeout_seconds=0.125), + otel=OTelConfig(enabled=False, shutdown_timeout_seconds=0.375), + ) + otel_timeouts: list[float] = [] + + class OTel: + def shutdown(self, timeout: float) -> None: + otel_timeouts.append(timeout) + + def fail_close(timeout: float) -> None: + assert timeout == 0.125 + raise RuntimeError("logging close failed") + + logger = logging.getLogger("tests.shutdown.cleanup") + logger.handlers.clear() + logger.propagate = False + handler = instrument_logging(logger, observed) + observed._otel = OTel() + monkeypatch.setattr(observed._delivery, "close", fail_close) + + observed.shutdown() + observed.shutdown() + + assert otel_timeouts == [0.375] + assert handler not in logger.handlers + assert observed.diagnostics.count("failure.logging.shutdown") == 1 + + +def test_shutdown_bounds_otel_with_its_own_timeout() -> None: + observed, _output = make_runtime( + logging=LoggingConfig(shutdown_timeout_seconds=0), + otel=OTelConfig(enabled=False, shutdown_timeout_seconds=0.01), + ) + started = threading.Event() + release = threading.Event() + + class BlockingOTel: + def shutdown(self, timeout: float) -> None: + assert timeout == 0.01 + started.set() + release.wait(1) + + observed._otel = BlockingOTel() + before = time.perf_counter() + observed.shutdown() + elapsed = time.perf_counter() - before + + assert started.is_set() + assert elapsed < 0.2 + assert observed.diagnostics.count("shutdown.timeout") == 1 + release.set() + + +def test_shutdown_contains_otel_coordinator_failure(monkeypatch) -> None: + observed, _output = make_runtime() + + def fail_bounded_shutdown(*_args, **_kwargs) -> None: + raise RuntimeError("could not start shutdown worker") + + monkeypatch.setattr( + "policyengine_observability.runtime._run_bounded", + fail_bounded_shutdown, + ) + + observed.shutdown() + + assert observed.diagnostics.count("failure.otel.shutdown") == 1 + + +@pytest.mark.parametrize( + ("logging_timeout", "otel_timeout", "expected_logging", "expected_otel"), + [ + ("invalid", float("inf"), 2.0, 3.0), + (-1.0, 100.0, 0.0, 60.0), + ], +) +def test_shutdown_timeout_configuration_is_safely_bounded( + logging_timeout, + otel_timeout, + expected_logging, + expected_otel, +) -> None: + observed, _output = make_runtime( + logging=LoggingConfig(shutdown_timeout_seconds=logging_timeout), + otel=OTelConfig( + enabled=False, + shutdown_timeout_seconds=otel_timeout, + ), + ) + observed_timeouts: list[tuple[str, float]] = [] + + class OTel: + def shutdown(self, timeout: float) -> None: + observed_timeouts.append(("otel", timeout)) + + observed._delivery.close = lambda timeout: observed_timeouts.append( + ("logging", timeout) + ) + observed._otel = OTel() + + observed.shutdown() + + assert observed_timeouts == [ + ("logging", expected_logging), + ("otel", expected_otel), + ] + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires POSIX fork") +def test_restart_after_snapshot_replaces_inherited_locked_state() -> None: + observed, _output = make_runtime() + observed._shutdown_lock.acquire() + observed.diagnostics._lock.acquire() + read_fd, write_fd = os.pipe() + child_pid = os.fork() + + if child_pid == 0: + os.close(read_fd) + try: + observed.restart_after_snapshot() + observed.diagnostics.increment("post_fork") + observed.shutdown() + os.write(write_fd, b"ok") + except BaseException as error: + os.write(write_fd, repr(error).encode()[:1_024]) + finally: + os.close(write_fd) + os._exit(0) + + os.close(write_fd) + result = b"" + try: + readable, _, _ = select.select([read_fd], [], [], 2) + if not readable: + os.kill(child_pid, signal.SIGKILL) + pytest.fail("child blocked on inherited observability state") + result = os.read(read_fd, 1_024) + finally: + observed.diagnostics._lock.release() + observed._shutdown_lock.release() + os.close(read_fd) + os.waitpid(child_pid, 0) + observed.shutdown() + + assert result == b"ok" + + +def test_restart_after_snapshot_rebuilds_process_local_components() -> None: + observed, _output = make_runtime() + observed.diagnostics.increment("before_snapshot") + observed.begin_request( + headers={REQUEST_ID_HEADER: "snapshotted-request"}, + method="GET", + route="/snapshot", + ) + inherited_delivery = observed._delivery + inherited_otel = observed._otel + + observed.restart_after_snapshot() + + assert observed._delivery is not inherited_delivery + assert observed._otel is not inherited_otel + assert observed.response_headers() == {} + assert observed.diagnostics.count("before_snapshot") == 0 + assert not observed._closed + observed.shutdown() + + def test_local_drop_and_export_diagnostics_increment_otel_metrics() -> None: observed, _output = make_runtime() calls: list[tuple[str, str, int]] = [] From 3db32432d8546f9cd28e8e633729caa508af7ec7 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:39:45 +0400 Subject: [PATCH 04/12] Parameterize Google Cloud deployment identities --- .gitignore | 1 + README.md | 12 +- changelog.d/29.breaking.md | 2 + deploy/gcp/README.md | 127 +++++++++++------- .../gcp/{alerts.yaml => alerts.template.yaml} | 2 +- deploy/gcp/collector/config.yaml | 2 +- .../{service.yaml => service.template.yaml} | 9 +- deploy/gcp/create_alerts.py | 9 +- ...dashboard.json => dashboard.template.json} | 2 +- deploy/gcp/{iam.yaml => iam.template.yaml} | 18 +-- ...routing.yaml => log-routing.template.yaml} | 8 +- deploy/gcp/render_deployment.py | 94 +++++++++++++ deploy/gcp/verify.sh | 14 +- deploy/gcp/verify_modal_wif.py | 56 ++++++-- deploy/gcp/verify_otel.py | 7 +- ....yaml => workload-inventory.template.yaml} | 24 ++-- docs/operations/api-v1-observability.md | 2 +- tests/test_deployment_assets.py | 92 ++++++++++++- 18 files changed, 377 insertions(+), 104 deletions(-) rename deploy/gcp/{alerts.yaml => alerts.template.yaml} (96%) rename deploy/gcp/collector/{service.yaml => service.template.yaml} (74%) rename deploy/gcp/{dashboard.json => dashboard.template.json} (98%) rename deploy/gcp/{iam.yaml => iam.template.yaml} (56%) rename deploy/gcp/{log-routing.yaml => log-routing.template.yaml} (89%) create mode 100644 deploy/gcp/render_deployment.py rename deploy/gcp/{workload-inventory.yaml => workload-inventory.template.yaml} (71%) diff --git a/.gitignore b/.gitignore index acb6a3f..234d2c8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ coverage.xml __pycache__/ *.py[cod] .DS_Store +/deploy/gcp/rendered/ diff --git a/README.md b/README.md index 003b2f8..7ff4001 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ Service and deployment identity are always explicit. Standard OpenTelemetry environment variables may supply OTLP transport settings. ```python +import os + from policyengine_observability import ( DeploymentIdentity, LoggingConfig, @@ -38,6 +40,7 @@ from policyengine_observability import ( configure, ) +central_project = os.environ["OBSERVABILITY_PROJECT_ID"] config = ObservabilityConfig( service=ServiceIdentity( name="policyengine-api", @@ -50,7 +53,7 @@ config = ObservabilityConfig( platform="google_cloud_run", region="us-central1", ), - google_cloud_project_id="policyengine-observability", + google_cloud_project_id=central_project, logging=LoggingConfig(shutdown_timeout_seconds=2), otel=OTelConfig( endpoint="https://COLLECTOR_HOST", @@ -203,21 +206,24 @@ Modal writes the same JSON immediately to standard output and may also send a copy through one bounded background Cloud Logging writer per process: ```python +import os + from policyengine_observability import ( GoogleCloudLoggingConfig, LoggingConfig, ) +central_project = os.environ["OBSERVABILITY_PROJECT_ID"] config = ObservabilityConfig.from_env( service=service, deployment=DeploymentIdentity( environment="production", platform="modal", ), - google_cloud_project_id="policyengine-observability", + google_cloud_project_id=central_project, logging=LoggingConfig( remote=GoogleCloudLoggingConfig( - project_id="policyengine-observability", + project_id=central_project, log_name="policyengine-api-v1-modal", queue_capacity=1_000, batch_size=100, diff --git a/changelog.d/29.breaking.md b/changelog.d/29.breaking.md index 1c70f05..4bb591e 100644 --- a/changelog.d/29.breaking.md +++ b/changelog.d/29.breaking.md @@ -6,3 +6,5 @@ OTLP export, and allowlisted cross-service correlation. Consumers must replace removed version 1 imports before upgrading. Logging and OTel now have separate shutdown timeout settings, shutdown failures are contained per subsystem, and process recovery replaces inherited locks and asynchronous delivery state. +Google Cloud deployment manifests now render from validated repository +variables, with the Modal workspace identifier supplied as a protected secret. diff --git a/deploy/gcp/README.md b/deploy/gcp/README.md index c2eadd6..108c605 100644 --- a/deploy/gcp/README.md +++ b/deploy/gcp/README.md @@ -1,9 +1,11 @@ # Google Cloud deployment plan This directory defines the centralized API v1 observability resources in the -`policyengine-observability` project. The workload boundary is defined in -[`workload-inventory.yaml`](workload-inventory.yaml). Applications absent from -that inventory receive no credentials or destination permissions. +project selected by `OBSERVABILITY_PROJECT_ID`. The workload boundary is +defined in +[`workload-inventory.template.yaml`](workload-inventory.template.yaml). +Applications absent from that inventory receive no credentials or destination +permissions. These files are a reviewable deployment plan. Applying them changes live IAM, Cloud Logging routing, Cloud Run, and monitoring resources and therefore @@ -15,11 +17,12 @@ requires an operator-approved deployment window. | --- | --- | | `collector/config.yaml` | OTLP gRPC receiver, bounded processors, and Google Telemetry API exporter for traces and metrics | | `collector/Dockerfile` | Google-built OTel Collector 0.160.0 plus the reviewed configuration | -| `collector/service.yaml` | Authenticated Cloud Run collector with fixed CPU, memory, concurrency, health checks, and scaling bounds | -| `log-routing.yaml` | Exact Cloud Run source sinks, restricted Modal direct-log sink, and `_Default` duplicate exclusion | -| `iam.yaml` | Collector and Modal service accounts, Cloud Run invokers, and a dedicated Modal API v1 identity provider | -| `dashboard.json` | Initial request, latency, error, dropped-item, and exporter-failure dashboard | -| `alerts.yaml` | Initial alert policy inputs | +| `collector/service.template.yaml` | Authenticated Cloud Run collector with fixed CPU, memory, concurrency, health checks, and scaling bounds | +| `log-routing.template.yaml` | Exact Cloud Run source sinks, restricted Modal direct-log sink, and `_Default` duplicate exclusion | +| `iam.template.yaml` | Collector and Modal service accounts, Cloud Run invokers, and a dedicated Modal API v1 identity provider | +| `dashboard.template.json` | Initial request, latency, error, dropped-item, and exporter-failure dashboard | +| `alerts.template.yaml` | Initial alert policy inputs | +| `render_deployment.py` | Validates deployment variables and writes private rendered files under `rendered/` | | `verify.sh` | Read-only resource and routing checks after deployment | The collector accepts traces and metrics. Application logs do not enter the @@ -34,7 +37,41 @@ to receive spans needed for this decision. ## Deployment order -### 1. Enable services +### 1. Configure and render deployment values + +Configure these GitHub Actions repository variables: + +- `OBSERVABILITY_PROJECT_ID` +- `OBSERVABILITY_PROJECT_NUMBER` +- `API_PROJECT_ID` +- `SIMULATION_ENTRY_PROJECT_ID` + +Configure `MODAL_WORKSPACE_ID` as a GitHub Actions repository secret. It is +private deployment metadata and must not be printed by workflows. + +Workflows that render or apply the deployment must map the values explicitly: + +```yaml +env: + OBSERVABILITY_PROJECT_ID: ${{ vars.OBSERVABILITY_PROJECT_ID }} + OBSERVABILITY_PROJECT_NUMBER: ${{ vars.OBSERVABILITY_PROJECT_NUMBER }} + API_PROJECT_ID: ${{ vars.API_PROJECT_ID }} + SIMULATION_ENTRY_PROJECT_ID: ${{ vars.SIMULATION_ENTRY_PROJECT_ID }} + MODAL_WORKSPACE_ID: ${{ secrets.MODAL_WORKSPACE_ID }} +``` + +For an operator-run deployment, set the same five values in the local process +without writing them to a tracked file, then render the templates: + +```bash +.venv/bin/python deploy/gcp/render_deployment.py +``` + +The renderer validates every value, reports only variable names, and writes +files with owner-only permissions under the ignored `deploy/gcp/rendered/` +directory. + +### 2. Enable services ```bash gcloud services enable \ @@ -48,23 +85,23 @@ gcloud services enable \ sts.googleapis.com \ telemetry.googleapis.com \ tracing.googleapis.com \ - --project=policyengine-observability + --project="${OBSERVABILITY_PROJECT_ID}" ``` -### 2. Create identities +### 3. Create identities -Create these service accounts in `policyengine-observability`: +Create these service accounts in the central project: ```text -policyengine-otel-collector@policyengine-observability.iam.gserviceaccount.com -policyengine-api-v1-modal@policyengine-observability.iam.gserviceaccount.com +policyengine-otel-collector@${OBSERVABILITY_PROJECT_ID}.iam.gserviceaccount.com +policyengine-api-v1-modal@${OBSERVABILITY_PROJECT_ID}.iam.gserviceaccount.com ``` -Grant only the roles listed in [`iam.yaml`](iam.yaml). The collector receives -`roles/telemetry.writer` and `roles/serviceusage.serviceUsageConsumer`. The Modal identity receives -`roles/logging.logWriter` and collector invocation permission. The four -existing Cloud Run identities in the inventory receive collector invocation -permission on the collector service only. +Grant only the roles listed in the rendered `iam.yaml`. The collector receives +`roles/telemetry.writer` and `roles/serviceusage.serviceUsageConsumer`. The +Modal identity receives `roles/logging.logWriter` and collector invocation +permission. The four existing Cloud Run identities in the inventory receive +collector invocation permission on the collector service only. Remove project-level `roles/logging.logWriter` bindings from every identity outside this inventory. Source-project logging service agents use conditional @@ -74,43 +111,43 @@ project-level log writer is present. Create a separate `modal-api-v1` workload identity pool and provider using the issuer, audience, mappings, workspace, environment, and application condition -in `iam.yaml`. Do not modify the existing `modal/modal` provider during this -deployment; it belongs to applications excluded from this change. Grant the -new provider permission to impersonate only the API v1 Modal service account. +in the rendered `iam.yaml`. Do not modify the existing `modal/modal` provider +during this deployment; it belongs to applications excluded from this change. +Grant the new provider permission to impersonate only the API v1 Modal service +account. Before enabling the provider, decode one production and one staging Modal identity token locally and confirm that `workspace_id`, `environment_name`, and `app_name` exactly match the reviewed condition. -### 3. Build and deploy the collector +### 4. Build and deploy the collector ```bash gcloud artifacts repositories create observability \ --repository-format=docker \ --location=us-central1 \ --immutable-tags \ - --project=policyengine-observability + --project="${OBSERVABILITY_PROJECT_ID}" gcloud builds submit deploy/gcp/collector \ - --tag=us-central1-docker.pkg.dev/policyengine-observability/observability/otel-collector:0.160.0-api-v1-1 \ - --project=policyengine-observability + --tag="us-central1-docker.pkg.dev/${OBSERVABILITY_PROJECT_ID}/observability/otel-collector:0.160.0-api-v1-1" \ + --project="${OBSERVABILITY_PROJECT_ID}" -gcloud run services replace deploy/gcp/collector/service.yaml \ +gcloud run services replace deploy/gcp/rendered/collector/service.yaml \ --region=us-central1 \ - --project=policyengine-observability + --project="${OBSERVABILITY_PROJECT_ID}" ``` -Apply `roles/run.invoker` bindings for the five identities listed in -`iam.yaml`. Do not grant unauthenticated invocation. Record the HTTPS service -URL as both `OTEL_EXPORTER_OTLP_ENDPOINT` and +Apply `roles/run.invoker` bindings for the five identities listed in the +rendered `iam.yaml`. Do not grant unauthenticated invocation. Record the HTTPS +service URL as both `OTEL_EXPORTER_OTLP_ENDPOINT` and `POLICYENGINE_OTEL_GOOGLE_AUDIENCE` in participating service configuration. -### 4. Configure log routing +### 5. Configure log routing Create one aggregated sink in each source project using the exact service and -schema filters in [`log-routing.yaml`](log-routing.yaml). Grant each generated -sink writer identity `roles/logging.bucketWriter` on the central -`policyengine-observability` bucket. +schema filters in the rendered `log-routing.yaml`. Grant each generated sink +writer identity `roles/logging.bucketWriter` on the central log bucket. Update `policyengine-observability-app-logs` to the listed direct-log filter. Add the listed exclusion to `_Default`; this prevents a direct Modal log from @@ -120,12 +157,12 @@ Cloud Audit Log exclusions. After routing one synthetic record per participating service, confirm each `insertId` exists exactly once in the central project. -### 5. Create dashboard and alerts +### 6. Create dashboard and alerts ```bash gcloud monitoring dashboards create \ - --config-from-file=deploy/gcp/dashboard.json \ - --project=policyengine-observability + --config-from-file=deploy/gcp/rendered/dashboard.json \ + --project="${OBSERVABILITY_PROJECT_ID}" ``` Create the API-ready alert policies with: @@ -138,7 +175,7 @@ The script is idempotent by policy display name. It leaves notification-channel configuration empty when the project has no channel; add operator-owned channel identifiers after creating the relevant email, Slack, or paging destination. -### 6. Verify before consumer deployment +### 7. Verify before consumer deployment ```bash bash deploy/gcp/verify.sh @@ -154,8 +191,8 @@ For an operator-run Cloud Run identity check, temporarily grant the operator ```bash .venv/bin/python deploy/gcp/verify_otel.py \ - --endpoint=https://policyengine-api-v1-otel-collector-790230211054.us-central1.run.app \ - --service-account=sim-entry-beta-runtime@policyengine-simulation-entry.iam.gserviceaccount.com + --endpoint="${POLICYENGINE_OTEL_GOOGLE_AUDIENCE}" \ + --service-account="sim-entry-beta-runtime@${SIMULATION_ENTRY_PROJECT_ID}.iam.gserviceaccount.com" ``` Remove the temporary operator binding immediately after the check. The script @@ -191,16 +228,16 @@ application deployment. The infrastructure portion of this runbook was applied and verified on 2026-09-22: -- the global `policyengine-observability` log bucket retains records for 30 +- the global central log bucket retains records for 30 days and has log analytics enabled; - exact source-project sinks route the two API services and the two simulation entry services to that bucket; - the authenticated collector runs in `us-central1` as `policyengine-api-v1-otel-collector`; - the dedicated `modal-api-v1` identity provider is active with the workspace, - environment, and application conditions in `iam.yaml`; -- the only project-level `roles/logging.logWriter` identity is - `policyengine-api-v1-modal`; + environment, and application conditions in the rendered `iam.yaml`; +- the only project-level `roles/logging.logWriter` identity is the API v1 + Modal service account; - the dashboard and six alert policies are present and enabled; and - the project currently has no alert notification channel, so the policies record incidents without sending email, Slack, or paging notifications. diff --git a/deploy/gcp/alerts.yaml b/deploy/gcp/alerts.template.yaml similarity index 96% rename from deploy/gcp/alerts.yaml rename to deploy/gcp/alerts.template.yaml index cfd617e..19fdbac 100644 --- a/deploy/gcp/alerts.yaml +++ b/deploy/gcp/alerts.template.yaml @@ -36,7 +36,7 @@ policies: - display_name: API v1 monthly log ingestion above 10 GiB condition_type: metric_threshold metric_type: logging.googleapis.com/billing/log_bucket_monthly_bytes_ingested - metric_label_log_bucket_id: policyengine-observability + metric_label_log_bucket_id: ${OBSERVABILITY_PROJECT_ID} comparison: COMPARISON_GT threshold: 10737418240 duration: 0s diff --git a/deploy/gcp/collector/config.yaml b/deploy/gcp/collector/config.yaml index b9adbb5..2fa3ced 100644 --- a/deploy/gcp/collector/config.yaml +++ b/deploy/gcp/collector/config.yaml @@ -12,7 +12,7 @@ processors: resource/destination: attributes: - key: gcp.project_id - value: policyengine-observability + value: ${env:OBSERVABILITY_PROJECT_ID} action: upsert tail_sampling: decision_wait: 30s diff --git a/deploy/gcp/collector/service.yaml b/deploy/gcp/collector/service.template.yaml similarity index 74% rename from deploy/gcp/collector/service.yaml rename to deploy/gcp/collector/service.template.yaml index 5b5316f..8cde2e4 100644 --- a/deploy/gcp/collector/service.yaml +++ b/deploy/gcp/collector/service.template.yaml @@ -2,7 +2,7 @@ apiVersion: serving.knative.dev/v1 kind: Service metadata: name: policyengine-api-v1-otel-collector - namespace: "790230211054" + namespace: "${OBSERVABILITY_PROJECT_NUMBER}" labels: cloud.googleapis.com/location: us-central1 annotations: @@ -18,10 +18,13 @@ spec: spec: containerConcurrency: 100 timeoutSeconds: 30 - serviceAccountName: policyengine-otel-collector@policyengine-observability.iam.gserviceaccount.com + serviceAccountName: policyengine-otel-collector@${OBSERVABILITY_PROJECT_ID}.iam.gserviceaccount.com containers: - name: collector - image: us-central1-docker.pkg.dev/policyengine-observability/observability/otel-collector:0.160.0-api-v1-1 + image: us-central1-docker.pkg.dev/${OBSERVABILITY_PROJECT_ID}/observability/otel-collector:0.160.0-api-v1-1 + env: + - name: OBSERVABILITY_PROJECT_ID + value: ${OBSERVABILITY_PROJECT_ID} ports: - name: h2c containerPort: 8080 diff --git a/deploy/gcp/create_alerts.py b/deploy/gcp/create_alerts.py index 025e98a..f31df88 100644 --- a/deploy/gcp/create_alerts.py +++ b/deploy/gcp/create_alerts.py @@ -3,12 +3,13 @@ from __future__ import annotations import json +import os import subprocess import urllib.parse import urllib.request from typing import Any -PROJECT = "policyengine-observability" +PROJECT = os.environ.get("OBSERVABILITY_PROJECT_ID", "") API_ROOT = f"https://monitoring.googleapis.com/v3/projects/{PROJECT}" @@ -158,7 +159,7 @@ def _policy( 'resource.type="global" AND ' 'metric.type="logging.googleapis.com/billing/' 'log_bucket_monthly_bytes_ingested" AND ' - 'metric.label."log_bucket_id"="policyengine-observability"' + f'metric.label."log_bucket_id"="{PROJECT}"' ), comparison="COMPARISON_GT", threshold=10_737_418_240, @@ -199,6 +200,10 @@ def _request( def main() -> None: + if not PROJECT: + raise SystemExit( + "Missing deployment variable: OBSERVABILITY_PROJECT_ID" + ) token = _token() existing: dict[str, str] = {} page_token = "" diff --git a/deploy/gcp/dashboard.json b/deploy/gcp/dashboard.template.json similarity index 98% rename from deploy/gcp/dashboard.json rename to deploy/gcp/dashboard.template.json index e525517..2fa07b2 100644 --- a/deploy/gcp/dashboard.json +++ b/deploy/gcp/dashboard.template.json @@ -151,7 +151,7 @@ { "timeSeriesQuery": { "timeSeriesFilter": { - "filter": "resource.type=\"global\" AND metric.type=\"logging.googleapis.com/billing/log_bucket_bytes_ingested\" AND metric.label.\"log_bucket_id\"=\"policyengine-observability\"", + "filter": "resource.type=\"global\" AND metric.type=\"logging.googleapis.com/billing/log_bucket_bytes_ingested\" AND metric.label.\"log_bucket_id\"=\"${OBSERVABILITY_PROJECT_ID}\"", "aggregation": { "alignmentPeriod": "3600s", "perSeriesAligner": "ALIGN_SUM", diff --git a/deploy/gcp/iam.yaml b/deploy/gcp/iam.template.yaml similarity index 56% rename from deploy/gcp/iam.yaml rename to deploy/gcp/iam.template.yaml index 87e06ac..062e90b 100644 --- a/deploy/gcp/iam.yaml +++ b/deploy/gcp/iam.template.yaml @@ -1,36 +1,36 @@ service_accounts: collector: - email: policyengine-otel-collector@policyengine-observability.iam.gserviceaccount.com + email: policyengine-otel-collector@${OBSERVABILITY_PROJECT_ID}.iam.gserviceaccount.com project_roles: - roles/telemetry.writer - roles/serviceusage.serviceUsageConsumer modal: - email: policyengine-api-v1-modal@policyengine-observability.iam.gserviceaccount.com + email: policyengine-api-v1-modal@${OBSERVABILITY_PROJECT_ID}.iam.gserviceaccount.com project_roles: - roles/logging.logWriter collector_roles: - roles/run.invoker collector_invokers: - - policyengine-api-cr-runtime@policyengine-api.iam.gserviceaccount.com - - policyengine-api-cr-staging@policyengine-api.iam.gserviceaccount.com - - sim-entry-prod-runtime@policyengine-simulation-entry.iam.gserviceaccount.com - - sim-entry-beta-runtime@policyengine-simulation-entry.iam.gserviceaccount.com - - policyengine-api-v1-modal@policyengine-observability.iam.gserviceaccount.com + - policyengine-api-cr-runtime@${API_PROJECT_ID}.iam.gserviceaccount.com + - policyengine-api-cr-staging@${API_PROJECT_ID}.iam.gserviceaccount.com + - sim-entry-prod-runtime@${SIMULATION_ENTRY_PROJECT_ID}.iam.gserviceaccount.com + - sim-entry-beta-runtime@${SIMULATION_ENTRY_PROJECT_ID}.iam.gserviceaccount.com + - policyengine-api-v1-modal@${OBSERVABILITY_PROJECT_ID}.iam.gserviceaccount.com modal_workload_identity: pool_id: modal-api-v1 provider_id: modal-api-v1 issuer: https://oidc.modal.com allowed_audience: oidc.modal.com - workspace_id: ac-AVO94gmhUEaFCuduiKdpRJ + workspace_id: ${MODAL_WORKSPACE_ID} attribute_mapping: google.subject: assertion.app_name attribute.app_name: assertion.app_name attribute.environment_name: assertion.environment_name attribute.workspace_id: assertion.workspace_id attribute_condition: >- - assertion.workspace_id == "ac-AVO94gmhUEaFCuduiKdpRJ" && + assertion.workspace_id == "${MODAL_WORKSPACE_ID}" && (assertion.environment_name == "main" || assertion.environment_name == "staging") && (assertion.app_name == "policyengine-simulation-gateway" || assertion.app_name.matches("^policyengine-simulation-py[0-9]+-[0-9]+-[0-9]+$")) diff --git a/deploy/gcp/log-routing.yaml b/deploy/gcp/log-routing.template.yaml similarity index 89% rename from deploy/gcp/log-routing.yaml rename to deploy/gcp/log-routing.template.yaml index 2e9a7b4..8d84ed8 100644 --- a/deploy/gcp/log-routing.yaml +++ b/deploy/gcp/log-routing.template.yaml @@ -1,18 +1,18 @@ destination: - project: policyengine-observability + project: ${OBSERVABILITY_PROJECT_ID} location: global - bucket: policyengine-observability + bucket: ${OBSERVABILITY_PROJECT_ID} direct_log_name: policyengine-api-v1-modal source_sinks: - - project: policyengine-api + - project: ${API_PROJECT_ID} name: api-v1-central-observability filter: >- resource.type="cloud_run_revision" AND (resource.labels.service_name="policyengine-api" OR resource.labels.service_name="policyengine-api-staging") AND jsonPayload.schema_version="policyengine.observability.v2" - - project: policyengine-simulation-entry + - project: ${SIMULATION_ENTRY_PROJECT_ID} name: api-v1-central-observability filter: >- resource.type="cloud_run_revision" AND diff --git a/deploy/gcp/render_deployment.py b/deploy/gcp/render_deployment.py new file mode 100644 index 0000000..b85613e --- /dev/null +++ b/deploy/gcp/render_deployment.py @@ -0,0 +1,94 @@ +"""Render Google Cloud deployment templates from validated environment values.""" + +from __future__ import annotations + +import argparse +import os +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +DEFAULT_OUTPUT = ROOT / "rendered" +PLACEHOLDER = re.compile(r"\$\{([A-Z][A-Z0-9_]*)\}") +TEMPLATES = ( + ("iam.template.yaml", "iam.yaml"), + ("workload-inventory.template.yaml", "workload-inventory.yaml"), + ("log-routing.template.yaml", "log-routing.yaml"), + ("alerts.template.yaml", "alerts.yaml"), + ("dashboard.template.json", "dashboard.json"), + ("collector/service.template.yaml", "collector/service.yaml"), +) +VALIDATORS = { + "OBSERVABILITY_PROJECT_ID": re.compile(r"[a-z][a-z0-9-]{4,28}[a-z0-9]"), + "OBSERVABILITY_PROJECT_NUMBER": re.compile(r"[1-9][0-9]{5,29}"), + "API_PROJECT_ID": re.compile(r"[a-z][a-z0-9-]{4,28}[a-z0-9]"), + "SIMULATION_ENTRY_PROJECT_ID": re.compile(r"[a-z][a-z0-9-]{4,28}[a-z0-9]"), + "MODAL_WORKSPACE_ID": re.compile(r"ac-[A-Za-z0-9_-]+"), +} + + +def _deployment_values() -> dict[str, str]: + missing = sorted(name for name in VALIDATORS if not os.getenv(name)) + if missing: + raise SystemExit("Missing deployment variables: " + ", ".join(missing)) + + values = {name: os.environ[name] for name in VALIDATORS} + invalid = sorted( + name + for name, pattern in VALIDATORS.items() + if pattern.fullmatch(values[name]) is None + ) + if invalid: + raise SystemExit("Invalid deployment variables: " + ", ".join(invalid)) + return values + + +def _render(content: str, values: dict[str, str]) -> str: + referenced = set(PLACEHOLDER.findall(content)) + unknown = sorted(referenced - values.keys()) + if unknown: + raise SystemExit("Unknown deployment variables: " + ", ".join(unknown)) + rendered = PLACEHOLDER.sub(lambda match: values[match.group(1)], content) + unresolved = sorted(set(PLACEHOLDER.findall(rendered))) + if unresolved: + raise SystemExit( + "Unresolved deployment variables: " + ", ".join(unresolved) + ) + return rendered + + +def render_deployment(output_directory: Path) -> None: + values = _deployment_values() + rendered_files = [] + for source_name, output_name in TEMPLATES: + source = ROOT / source_name + rendered_files.append( + (output_name, _render(source.read_text(), values)) + ) + + for output_name, content in rendered_files: + destination = output_directory / output_name + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(content) + destination.chmod(0o600) + + variable_names = ", ".join(sorted(values)) + print( + f"Rendered {len(TEMPLATES)} deployment files using: {variable_names}" + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--output-dir", + type=Path, + default=DEFAULT_OUTPUT, + help="Destination for rendered files (default: deploy/gcp/rendered)", + ) + arguments = parser.parse_args() + render_deployment(arguments.output_dir) + + +if __name__ == "__main__": + main() diff --git a/deploy/gcp/verify.sh b/deploy/gcp/verify.sh index eaed4c5..a14188a 100644 --- a/deploy/gcp/verify.sh +++ b/deploy/gcp/verify.sh @@ -1,11 +1,15 @@ #!/usr/bin/env bash set -euo pipefail -project="policyengine-observability" +: "${OBSERVABILITY_PROJECT_ID:?Missing OBSERVABILITY_PROJECT_ID}" +: "${API_PROJECT_ID:?Missing API_PROJECT_ID}" +: "${SIMULATION_ENTRY_PROJECT_ID:?Missing SIMULATION_ENTRY_PROJECT_ID}" + +project="${OBSERVABILITY_PROJECT_ID}" region="us-central1" collector="policyengine-api-v1-otel-collector" -gcloud logging buckets describe policyengine-observability \ +gcloud logging buckets describe "${OBSERVABILITY_PROJECT_ID}" \ --location=global \ --project="${project}" \ --format='value(name,retentionDays,analyticsEnabled)' @@ -26,14 +30,14 @@ log_writers="$( --filter='bindings.role=roles/logging.logWriter' \ --format='value(bindings.members)' )" -expected_log_writer='serviceAccount:policyengine-api-v1-modal@policyengine-observability.iam.gserviceaccount.com' +expected_log_writer="serviceAccount:policyengine-api-v1-modal@${OBSERVABILITY_PROJECT_ID}.iam.gserviceaccount.com" if [[ "${log_writers}" != "${expected_log_writer}" ]]; then echo "Unexpected project-level Cloud Logging writers: ${log_writers}" >&2 exit 1 fi gcloud logging views get-iam-policy _AllLogs \ - --bucket=policyengine-observability \ + --bucket="${OBSERVABILITY_PROJECT_ID}" \ --location=global \ --project="${project}" \ --format=json @@ -44,7 +48,7 @@ gcloud iam workload-identity-pools providers describe modal-api-v1 \ --project="${project}" \ --format='yaml(state,attributeCondition,attributeMapping,oidc)' -for source_project in policyengine-api policyengine-simulation-entry; do +for source_project in "${API_PROJECT_ID}" "${SIMULATION_ENTRY_PROJECT_ID}"; do gcloud logging sinks describe api-v1-central-observability \ --project="${source_project}" \ --format='yaml(destination,filter,writerIdentity)' diff --git a/deploy/gcp/verify_modal_wif.py b/deploy/gcp/verify_modal_wif.py index 80aab36..7c8aaaa 100644 --- a/deploy/gcp/verify_modal_wif.py +++ b/deploy/gcp/verify_modal_wif.py @@ -9,6 +9,7 @@ from __future__ import annotations import base64 +import hashlib import json import os import time @@ -18,12 +19,8 @@ import modal -PROJECT = "policyengine-observability" -PROJECT_NUMBER = "790230211054" POOL = "modal-api-v1" PROVIDER = "modal-api-v1" -SERVICE_ACCOUNT = "policyengine-api-v1-modal@policyengine-observability.iam.gserviceaccount.com" -COLLECTOR = "https://policyengine-api-v1-otel-collector-790230211054.us-central1.run.app" APP_NAME = os.environ.get( "POLICYENGINE_WIF_TEST_APP", "policyengine-observability-wif-denied-test", @@ -32,6 +29,13 @@ app = modal.App(APP_NAME) +def _required_environment(name: str) -> str: + value = os.getenv(name) + if not value: + raise SystemExit(f"Missing deployment variable: {name}") + return value + + def _jwt_claims(token: str) -> dict[str, object]: payload = token.split(".")[1] payload += "=" * (-len(payload) % 4) @@ -84,13 +88,23 @@ def _error_result(error: urllib.error.HTTPError) -> dict[str, object]: @app.function(timeout=60) -def verify_identity() -> dict[str, object]: +def verify_identity( + project: str, + project_number: str, + workspace_id_digest: str, +) -> dict[str, object]: + service_account = ( + f"policyengine-api-v1-modal@{project}.iam.gserviceaccount.com" + ) + collector = ( + "https://policyengine-api-v1-otel-collector-" + f"{project_number}.us-central1.run.app" + ) identity_token = os.environ["MODAL_IDENTITY_TOKEN"] claims = _jwt_claims(identity_token) safe_claims = { key: claims.get(key) for key in ( - "workspace_id", "environment_name", "app_name", "function_name", @@ -98,9 +112,14 @@ def verify_identity() -> dict[str, object]: "iss", ) } + workspace_id = str(claims.get("workspace_id", "")) + workspace_id_matches = ( + hashlib.sha256(workspace_id.encode()).hexdigest() + == workspace_id_digest + ) audience = ( "//iam.googleapis.com/projects/" - f"{PROJECT_NUMBER}/locations/global/workloadIdentityPools/{POOL}" + f"{project_number}/locations/global/workloadIdentityPools/{POOL}" f"/providers/{PROVIDER}" ) try: @@ -120,13 +139,14 @@ def verify_identity() -> dict[str, object]: except urllib.error.HTTPError as error: return { "claims": safe_claims, + "workspace_id_matches": workspace_id_matches, "token_exchange": _error_result(error), } federated_token = str(sts_payload["access_token"]) service_account_url = ( "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/" - f"{SERVICE_ACCOUNT}" + f"{service_account}" ) access_status, access_payload = _post_json( f"{service_account_url}:generateAccessToken", @@ -140,12 +160,12 @@ def verify_identity() -> dict[str, object]: identity_status, identity_payload = _post_json( f"{service_account_url}:generateIdToken", - {"audience": COLLECTOR, "includeEmail": True}, + {"audience": collector, "includeEmail": True}, bearer_token=federated_token, ) collector_identity_token = str(identity_payload["token"]) collector_request = urllib.request.Request( - COLLECTOR, + collector, headers={"Authorization": f"Bearer {collector_identity_token}"}, ) try: @@ -158,10 +178,10 @@ def verify_identity() -> dict[str, object]: logging_status, _ = _post_json( "https://logging.googleapis.com/v2/entries:write", { - "logName": f"projects/{PROJECT}/logs/policyengine-api-v1-modal", + "logName": f"projects/{project}/logs/policyengine-api-v1-modal", "resource": { "type": "global", - "labels": {"project_id": PROJECT}, + "labels": {"project_id": project}, }, "entries": [ { @@ -180,6 +200,7 @@ def verify_identity() -> dict[str, object]: ) return { "claims": safe_claims, + "workspace_id_matches": workspace_id_matches, "token_exchange": {"http_status": 200}, "service_account_access": {"http_status": access_status}, "service_account_identity": {"http_status": identity_status}, @@ -191,4 +212,13 @@ def verify_identity() -> dict[str, object]: @app.local_entrypoint() def main() -> None: - print(json.dumps(getattr(verify_identity, "remote")(), sort_keys=True)) + project = _required_environment("OBSERVABILITY_PROJECT_ID") + project_number = _required_environment("OBSERVABILITY_PROJECT_NUMBER") + workspace_id = _required_environment("MODAL_WORKSPACE_ID") + workspace_id_digest = hashlib.sha256(workspace_id.encode()).hexdigest() + result = getattr(verify_identity, "remote")( + project, + project_number, + workspace_id_digest, + ) + print(json.dumps(result, sort_keys=True)) diff --git a/deploy/gcp/verify_otel.py b/deploy/gcp/verify_otel.py index 3acde9b..08a9e8a 100644 --- a/deploy/gcp/verify_otel.py +++ b/deploy/gcp/verify_otel.py @@ -4,6 +4,7 @@ import argparse import json +import os import secrets import subprocess import time @@ -39,7 +40,7 @@ from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor -PROJECT = "policyengine-observability" +PROJECT = os.environ.get("OBSERVABILITY_PROJECT_ID", "") METRIC_NAME = "policyengine.verification.counter" METRIC_TYPE = f"prometheus.googleapis.com/{METRIC_NAME}/counter" @@ -219,6 +220,10 @@ def _wait_for_storage( def main() -> int: + if not PROJECT: + raise SystemExit( + "Missing deployment variable: OBSERVABILITY_PROJECT_ID" + ) parser = argparse.ArgumentParser() parser.add_argument("--endpoint", required=True) parser.add_argument("--service-account", required=True) diff --git a/deploy/gcp/workload-inventory.yaml b/deploy/gcp/workload-inventory.template.yaml similarity index 71% rename from deploy/gcp/workload-inventory.yaml rename to deploy/gcp/workload-inventory.template.yaml index ce89ae2..f74db60 100644 --- a/deploy/gcp/workload-inventory.yaml +++ b/deploy/gcp/workload-inventory.template.yaml @@ -1,44 +1,44 @@ schema_version: policyengine.observability.workloads.v1 destination_project: - project_id: policyengine-observability - project_number: "790230211054" - log_bucket: policyengine-observability + project_id: ${OBSERVABILITY_PROJECT_ID} + project_number: "${OBSERVABILITY_PROJECT_NUMBER}" + log_bucket: ${OBSERVABILITY_PROJECT_ID} log_bucket_location: global log_retention_days: 30 cloud_run: - service_name: policyengine-api - source_project: policyengine-api + source_project: ${API_PROJECT_ID} region: us-central1 environment: production - service_account: policyengine-api-cr-runtime@policyengine-api.iam.gserviceaccount.com + service_account: policyengine-api-cr-runtime@${API_PROJECT_ID}.iam.gserviceaccount.com signals: [logs, traces, metrics] include_tagged_revisions: true - service_name: policyengine-api-staging - source_project: policyengine-api + source_project: ${API_PROJECT_ID} region: us-central1 environment: staging - service_account: policyengine-api-cr-staging@policyengine-api.iam.gserviceaccount.com + service_account: policyengine-api-cr-staging@${API_PROJECT_ID}.iam.gserviceaccount.com signals: [logs, traces, metrics] include_tagged_revisions: true - service_name: policyengine-simulation-entry - source_project: policyengine-simulation-entry + source_project: ${SIMULATION_ENTRY_PROJECT_ID} region: us-central1 environment: production - service_account: sim-entry-prod-runtime@policyengine-simulation-entry.iam.gserviceaccount.com + service_account: sim-entry-prod-runtime@${SIMULATION_ENTRY_PROJECT_ID}.iam.gserviceaccount.com signals: [logs, traces, metrics] include_tagged_revisions: true - service_name: policyengine-simulation-entry-staging - source_project: policyengine-simulation-entry + source_project: ${SIMULATION_ENTRY_PROJECT_ID} region: us-central1 environment: staging - service_account: sim-entry-beta-runtime@policyengine-simulation-entry.iam.gserviceaccount.com + service_account: sim-entry-beta-runtime@${SIMULATION_ENTRY_PROJECT_ID}.iam.gserviceaccount.com signals: [logs, traces, metrics] include_tagged_revisions: true modal: workspace_name: policyengine - workspace_id: ac-AVO94gmhUEaFCuduiKdpRJ + workspace_id: ${MODAL_WORKSPACE_ID} environments: - name: main deployment_environment: production diff --git a/docs/operations/api-v1-observability.md b/docs/operations/api-v1-observability.md index cfebaf5..58960ac 100644 --- a/docs/operations/api-v1-observability.md +++ b/docs/operations/api-v1-observability.md @@ -3,7 +3,7 @@ ## Scope The machine-readable workload inventory is -[`deploy/gcp/workload-inventory.yaml`](../../deploy/gcp/workload-inventory.yaml). +[`deploy/gcp/workload-inventory.template.yaml`](../../deploy/gcp/workload-inventory.template.yaml). Only the listed `policyengine-api`, simulation entry, simulation gateway, and versioned simulation executor workloads participate. diff --git a/tests/test_deployment_assets.py b/tests/test_deployment_assets.py index 791d4d6..f74efb1 100644 --- a/tests/test_deployment_assets.py +++ b/tests/test_deployment_assets.py @@ -1,7 +1,10 @@ from __future__ import annotations import json +import os +import stat import subprocess +import sys from pathlib import Path ROOT = Path(__file__).parents[1] @@ -9,7 +12,7 @@ def test_dashboard_is_valid_json_with_required_signals() -> None: - dashboard = json.loads((DEPLOY / "dashboard.json").read_text()) + dashboard = json.loads((DEPLOY / "dashboard.template.json").read_text()) serialized = json.dumps(dashboard) assert "policyengine.request.count" in serialized assert "policyengine.request.duration" in serialized @@ -29,8 +32,8 @@ def test_collector_accepts_only_traces_and_metrics() -> None: def test_authorization_assets_exclude_unrelated_applications() -> None: - iam = (DEPLOY / "iam.yaml").read_text() - routing = (DEPLOY / "log-routing.yaml").read_text() + iam = (DEPLOY / "iam.template.yaml").read_text() + routing = (DEPLOY / "log-routing.template.yaml").read_text() for excluded in ( "policyengine-household-api", "policyengine-uk-chat", @@ -46,6 +49,89 @@ def test_authorization_assets_exclude_unrelated_applications() -> None: assert 'jsonPayload."service.namespace"' in routing +def test_deployment_templates_use_environment_placeholders() -> None: + templates = [ + DEPLOY / "iam.template.yaml", + DEPLOY / "workload-inventory.template.yaml", + DEPLOY / "log-routing.template.yaml", + DEPLOY / "alerts.template.yaml", + DEPLOY / "dashboard.template.json", + DEPLOY / "collector" / "service.template.yaml", + ] + content = "\n".join(path.read_text() for path in templates) + for variable in ( + "OBSERVABILITY_PROJECT_ID", + "OBSERVABILITY_PROJECT_NUMBER", + "API_PROJECT_ID", + "SIMULATION_ENTRY_PROJECT_ID", + "MODAL_WORKSPACE_ID", + ): + assert f"${{{variable}}}" in content + assert "workspace_id: ac-" not in content + + +def test_deployment_renderer_validates_and_does_not_print_values( + tmp_path: Path, +) -> None: + values = { + "OBSERVABILITY_PROJECT_ID": "central-observability", + "OBSERVABILITY_PROJECT_NUMBER": "123456789012", + "API_PROJECT_ID": "api-project", + "SIMULATION_ENTRY_PROJECT_ID": "simulation-entry-project", + "MODAL_WORKSPACE_ID": "ac-private-workspace", + } + environment = os.environ.copy() + environment.update(values) + result = subprocess.run( + [ + sys.executable, + str(DEPLOY / "render_deployment.py"), + "--output-dir", + str(tmp_path), + ], + check=True, + capture_output=True, + text=True, + env=environment, + ) + + assert all(value not in result.stdout for value in values.values()) + rendered_iam = (tmp_path / "iam.yaml").read_text() + assert "policyengine-otel-collector@central-observability" in rendered_iam + assert 'assertion.workspace_id == "ac-private-workspace"' in rendered_iam + assert stat.S_IMODE((tmp_path / "iam.yaml").stat().st_mode) == 0o600 + json.loads((tmp_path / "dashboard.json").read_text()) + + +def test_deployment_renderer_rejects_missing_values(tmp_path: Path) -> None: + environment = os.environ.copy() + for variable in ( + "OBSERVABILITY_PROJECT_ID", + "OBSERVABILITY_PROJECT_NUMBER", + "API_PROJECT_ID", + "SIMULATION_ENTRY_PROJECT_ID", + "MODAL_WORKSPACE_ID", + ): + environment.pop(variable, None) + + result = subprocess.run( + [ + sys.executable, + str(DEPLOY / "render_deployment.py"), + "--output-dir", + str(tmp_path), + ], + check=False, + capture_output=True, + text=True, + env=environment, + ) + + assert result.returncode != 0 + assert "Missing deployment variables:" in result.stderr + assert not list(tmp_path.iterdir()) + + def test_verification_script_has_valid_shell_syntax() -> None: subprocess.run( ["bash", "-n", str(DEPLOY / "verify.sh")], From 67681f28d50fedc37597eac10b23bd3362476015 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 22 Sep 2026 19:58:00 +0400 Subject: [PATCH 05/12] Make observability destinations consumer-configurable --- README.md | 274 +++++++++------- changelog.d/29.breaking.md | 16 +- deploy/gcp/README.md | 251 --------------- deploy/gcp/alerts.template.yaml | 42 --- deploy/gcp/collector/Dockerfile | 6 - deploy/gcp/collector/config.yaml | 71 ---- deploy/gcp/collector/service.template.yaml | 48 --- deploy/gcp/create_alerts.py | 255 --------------- deploy/gcp/dashboard.template.json | 183 ----------- deploy/gcp/iam.template.yaml | 36 --- deploy/gcp/log-routing.template.yaml | 37 --- deploy/gcp/render_deployment.py | 94 ------ deploy/gcp/verify.sh | 63 ---- deploy/gcp/verify_modal_wif.py | 224 ------------- deploy/gcp/verify_otel.py | 258 --------------- deploy/gcp/workload-inventory.template.yaml | 70 ---- .../engineering/skills/repository-guidance.md | 29 +- docs/operations/api-v1-observability.md | 153 --------- policyengine_observability/__init__.py | 22 +- policyengine_observability/config.py | 210 +++++++----- policyengine_observability/delivery.py | 304 ++++++++++-------- .../destinations/__init__.py | 25 ++ .../destinations/base.py | 92 ++++++ .../destinations/google_cloud.py | 134 ++++++++ .../destinations/stdout.py | 50 +++ policyengine_observability/google_auth.py | 107 ++++++ policyengine_observability/otel.py | 135 ++------ policyengine_observability/runtime.py | 8 +- policyengine_observability/schema.py | 12 - tests/conftest.py | 9 +- tests/test_config_schema.py | 58 ++-- tests/test_delivery.py | 114 ++++--- tests/test_deployment_assets.py | 141 -------- tests/test_destination_strategies.py | 198 ++++++++++++ tests/test_diagnostics_public_api.py | 2 +- tests/test_google_credentials.py | 6 +- tests/test_otel.py | 49 +-- tests/test_otel_destinations.py | 55 ++++ tests/test_runtime.py | 2 +- 39 files changed, 1314 insertions(+), 2529 deletions(-) delete mode 100644 deploy/gcp/README.md delete mode 100644 deploy/gcp/alerts.template.yaml delete mode 100644 deploy/gcp/collector/Dockerfile delete mode 100644 deploy/gcp/collector/config.yaml delete mode 100644 deploy/gcp/collector/service.template.yaml delete mode 100644 deploy/gcp/create_alerts.py delete mode 100644 deploy/gcp/dashboard.template.json delete mode 100644 deploy/gcp/iam.template.yaml delete mode 100644 deploy/gcp/log-routing.template.yaml delete mode 100644 deploy/gcp/render_deployment.py delete mode 100644 deploy/gcp/verify.sh delete mode 100644 deploy/gcp/verify_modal_wif.py delete mode 100644 deploy/gcp/verify_otel.py delete mode 100644 deploy/gcp/workload-inventory.template.yaml delete mode 100644 docs/operations/api-v1-observability.md create mode 100644 policyengine_observability/destinations/__init__.py create mode 100644 policyengine_observability/destinations/base.py create mode 100644 policyengine_observability/destinations/google_cloud.py create mode 100644 policyengine_observability/destinations/stdout.py create mode 100644 policyengine_observability/google_auth.py delete mode 100644 tests/test_deployment_assets.py create mode 100644 tests/test_destination_strategies.py create mode 100644 tests/test_otel_destinations.py diff --git a/README.md b/README.md index 7ff4001..83a1c32 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,14 @@ # policyengine-observability -`policyengine-observability` provides structured application logs, +`policyengine-observability` provides a shared runtime for structured logs, OpenTelemetry traces and metrics, request propagation, and framework request -instrumentation for PolicyEngine services. +instrumentation. Each application owns its service identity, attribute policy, +logging destinations, OTLP endpoints, and credentials. -Version 2 has one explicit ownership model: `configure(config)` returns a -runtime, and every adapter or manual operation receives that runtime. The -package does not install a mutable global runtime, infer a remote destination, -or contact a remote service during configuration. +Version 2 uses an explicitly owned runtime. `configure(config)` returns that +runtime, and every adapter or manual operation receives it. Configuration does +not select a destination from the deployment platform or contact a remote +service. ## Install @@ -25,26 +26,25 @@ are imported only when configured or called. ## Configure a runtime -Service and deployment identity are always explicit. Standard OpenTelemetry -environment variables may supply OTLP transport settings. +Service and deployment identity are explicit. Logging and OTel use separate +configuration so they can write to different stores. ```python -import os - from policyengine_observability import ( DeploymentIdentity, LoggingConfig, ObservabilityConfig, OTelConfig, + OTLPExporterConfig, ServiceIdentity, + StdoutLogDestination, configure, ) -central_project = os.environ["OBSERVABILITY_PROJECT_ID"] config = ObservabilityConfig( service=ServiceIdentity( - name="policyengine-api", - namespace="policyengine.api-v1", + name="example-api", + namespace="policyengine.example", version="1.2.3", role="api", ), @@ -53,35 +53,137 @@ config = ObservabilityConfig( platform="google_cloud_run", region="us-central1", ), - google_cloud_project_id=central_project, - logging=LoggingConfig(shutdown_timeout_seconds=2), + logging=LoggingConfig( + destinations=(StdoutLogDestination(),), + ), otel=OTelConfig( - endpoint="https://COLLECTOR_HOST", - google_audience="https://COLLECTOR_HOST", - shutdown_timeout_seconds=3, + traces=OTLPExporterConfig(endpoint="collector:4317"), + metrics=OTLPExporterConfig(endpoint="collector:4317"), ), + application_attribute_keys=frozenset({"country_id", "backend"}), + dispatch_attribute_keys=frozenset({"job_id", "run_id"}), ) runtime = configure(config) ``` -Cloud Run writes one-line JSON to standard output. It does not use the Cloud -Logging API from the application process. When no OTLP endpoint is configured, -the runtime retains local trace context for log correlation and creates no -remote exporter. +The default logging destination is one-line JSON on standard output. An OTel +runtime without an exporter still creates local trace context for log +correlation. It does not send traces or metrics remotely. + +## Logging destinations + +Application code emits a provider-neutral record. `LoggingConfig` selects one +or more destination strategies when the runtime starts. + +```python +from policyengine_observability import ( + GoogleCloudLogDestination, + GoogleCloudLogFormatter, +) + +logging = LoggingConfig( + destinations=( + StdoutLogDestination( + formatter=GoogleCloudLogFormatter("trace-project"), + ), + GoogleCloudLogDestination( + project_id="logging-project", + log_name="example-api", + queue_capacity=1_000, + batch_size=100, + write_timeout_seconds=5, + ), + ), +) +``` + +The formatter adds Cloud Logging trace-correlation fields only to the output +it formats. The canonical record retains the portable `trace_id`, `span_id`, +and `trace_sampled` fields. + +Every queued destination has its own bounded queue and worker. A blocked or +failing destination cannot delay another destination or the application +operation that emitted the record. Queue saturation drops the newest record +and records a local diagnostic. + +### Custom destinations + +Use `CustomLogDestination` for a writer owned by an application or another +package: + +```python +from policyengine_observability import CustomLogDestination + +logging = LoggingConfig( + destinations=( + CustomLogDestination( + name="internal-log-store", + writer_factory=lambda: InternalLogWriter(), + delivery="queued", + ), + ), +) +``` + +The writer implements `write(record)`. It may optionally implement +`write_many(records)` and `close()`. A separate integration package can also +provide a class implementing `LogDestinationStrategy`. Network writers should +always use `delivery="queued"`. + +## OTLP destinations and authentication -The supported OTel environment settings include: +Traces and metrics have independent exporter configurations: + +```python +otel = OTelConfig( + traces=OTLPExporterConfig( + endpoint="https://trace-collector.example", + protocol="http/protobuf", + headers=(("x-api-key", "trace-key"),), + ), + metrics=OTLPExporterConfig( + endpoint="metrics-collector.example:4317", + protocol="grpc", + headers=(("x-api-key", "metric-key"),), + ), +) +``` + +For a Google ID-token protected collector, select the authentication strategy +explicitly: + +```python +from policyengine_observability import GoogleIdTokenAuth + +traces = OTLPExporterConfig( + endpoint="collector.example:443", + auth=GoogleIdTokenAuth("https://collector.example"), +) +``` + +`ObservabilityConfig.from_env` reads the standard common and signal-specific +OTel settings, including: ```bash -OTEL_EXPORTER_OTLP_ENDPOINT=https://COLLECTOR_HOST +OTEL_EXPORTER_OTLP_ENDPOINT=collector.example:4317 +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=trace-collector.example:4317 +OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=metric-collector.example:4317 OTEL_EXPORTER_OTLP_PROTOCOL=grpc -OTEL_TRACES_SAMPLER_ARG=1.0 -POLICYENGINE_OTEL_GOOGLE_AUDIENCE=https://COLLECTOR_HOST +OTEL_EXPORTER_OTLP_TRACES_HEADERS=x-api-key=trace-key +OTEL_EXPORTER_OTLP_METRICS_HEADERS=x-api-key=metric-key ``` +`POLICYENGINE_OTEL_GOOGLE_AUDIENCE` selects `GoogleIdTokenAuth` for both +signals. `POLICYENGINE_OTEL_TRACES_GOOGLE_AUDIENCE` and +`POLICYENGINE_OTEL_METRICS_GOOGLE_AUDIENCE` override it per signal. + +Applications can share a collector by configuring the same endpoint and +credentials. Another application can use a separate collector or any +OTLP-compatible service without changing this package. + ## Flask and FastAPI -Install a framework adapter once while constructing the application. Routes -do not need observability decorators. +Install a framework adapter once while constructing the application: ```python from flask import Flask @@ -104,11 +206,10 @@ Both adapters extract W3C trace context and record, record request metrics, and clear context-local state. Repeated calls reuse the first runtime associated with the application. -## Manual operations and spans +## Application instrumentation -Use an operation for one complete non-HTTP invocation. It emits one completion -record and records duration, count, and error metrics. Use spans for child -steps; child spans do not emit additional operation completion records. +The package instruments framework and transport mechanics. Applications own +the names and boundaries of their domain operations. ```python with runtime.operation( @@ -129,12 +230,8 @@ async def calculate(simulation): return await simulation.calculate() ``` -Function arguments and return values are never captured. Telemetry failures do -not change the function return value or replace an application exception. - -## Events, application logs, and handled exceptions - -The runtime accepts only configured attribute keys and scalar values. +Function arguments and return values are never captured. The runtime accepts +only application-configured attribute keys and scalar values. ```python runtime.set_context(auth_result="accepted", simulation_id="sim-123") @@ -146,26 +243,18 @@ except ValueError as error: runtime.record_exception(error, handled=True) ``` -To route standard-library logs through the versioned JSON schema, instrument a -specific logger or configure root capture explicitly: +## Python logging and HTTP propagation + +Instrument a specific Python logger or configure root capture explicitly: ```python import logging from policyengine_observability import instrument_logging -logger = logging.getLogger("policyengine.api") +logger = logging.getLogger("policyengine.example") instrument_logging(logger, runtime) -logger.info( - "Simulation accepted", - extra={"policyengine_attributes": {"backend": "modal"}}, -) ``` -The handler ignores Google, gRPC, OTel, and this package's own loggers to avoid -recursive export failures. - -## Outbound HTTP and asynchronous dispatch - Only the supplied HTTPX client is modified: ```python @@ -179,16 +268,12 @@ instrument_httpx(client, runtime) The request hook injects active W3C context and the PolicyEngine request ID. Other clients in the process remain unchanged. -For a queued simulation, serialize the bounded correlation context with the -job request: +For asynchronous dispatch, serialize the bounded correlation context with the +job request and restore it around the worker operation: ```python request.observability_context = runtime.capture_context() -``` - -Then restore it around the worker invocation: -```python with runtime.operation( "simulation.run", remote_context=request.observability_context, @@ -196,50 +281,11 @@ with runtime.operation( return run_simulation(request) ``` -A direct continuation that starts within five minutes uses the dispatched span -as its parent. An older invocation, independent retry, or aggregate operation -starts a new trace with a link to the dispatch span. +## Process restoration and shutdown -## Modal logging and snapshots - -Modal writes the same JSON immediately to standard output and may also send a -copy through one bounded background Cloud Logging writer per process: - -```python -import os - -from policyengine_observability import ( - GoogleCloudLoggingConfig, - LoggingConfig, -) - -central_project = os.environ["OBSERVABILITY_PROJECT_ID"] -config = ObservabilityConfig.from_env( - service=service, - deployment=DeploymentIdentity( - environment="production", - platform="modal", - ), - google_cloud_project_id=central_project, - logging=LoggingConfig( - remote=GoogleCloudLoggingConfig( - project_id=central_project, - log_name="policyengine-api-v1-modal", - queue_capacity=1_000, - batch_size=100, - write_timeout_seconds=5, - ) - ), -) -runtime = configure(config) -``` - -Remote logging starts no client until the background worker receives its first -record. The calling thread uses `put_nowait`; a full queue drops the newest -record and records a local counter. - -After a Modal memory snapshot restores, rebuild process-local locks, context, -queues, threads, credentials, and exporters before accepting work: +After a process image or memory snapshot is restored, rebuild process-local +locks, context, queues, threads, credentials, and exporters before accepting +work: ```python @modal.enter(snap=False) @@ -247,32 +293,22 @@ def restore_process_state(self): self.runtime.restart_after_snapshot() ``` -## Failure behavior and shutdown - -Remote telemetry delivery is best effort. Exporters have bounded queues, -batches, retries, network deadlines, and shutdown periods. Missing credentials, -invalid configuration, unavailable DNS, denied permissions, collector failure, -queue saturation, and shutdown timeouts produce rate-limited JSON diagnostics -on standard error. Those diagnostics do not enter the failing exporter. - Call `runtime.shutdown()` during orderly process shutdown. Logging and OTel -each use the timeout configured on their respective configuration objects. A -failure in either subsystem is reported locally and does not prevent cleanup -of the other subsystem. The call is safe to repeat. - -The operating policy, workload allowlist, Google Cloud deployment plan, and -rollback procedure are in -[`docs/operations/api-v1-observability.md`](docs/operations/api-v1-observability.md) -and [`deploy/gcp/README.md`](deploy/gcp/README.md). +each use their own timeout. Missing dependencies, invalid configuration, +credential failures, unavailable destinations, queue saturation, exporter +errors, and shutdown timeouts produce rate-limited diagnostics on standard +error. These failures do not change application responses, return values, or +exceptions. ## Release workflow Changes include a Towncrier fragment in `changelog.d/`. Pull requests run Ruff, -tests, and coverage checks. The release workflow builds distributions and -publishes through PyPI trusted publishing. +tests, type checks, and coverage checks. The release workflow builds +distributions and publishes through PyPI trusted publishing. ## License Code in this repository is released under the [MIT License](LICENSE). Original -text and figures are released under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) -with attribution to PolicyEngine. Third-party materials retain their terms. +text and figures are released under +[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) with attribution to +PolicyEngine. Third-party materials retain their terms. diff --git a/changelog.d/29.breaking.md b/changelog.d/29.breaking.md index 4bb591e..87102e7 100644 --- a/changelog.d/29.breaking.md +++ b/changelog.d/29.breaking.md @@ -1,10 +1,10 @@ Replace the implicit global runtime and version 1 decorators with an explicit runtime returned by `configure`. Version 2 adds runtime-bound operations and -spans, idempotent Flask and FastAPI request adapters, explicit HTTPX and Python -logging instrumentation, bounded Modal Cloud Logging delivery, authenticated -OTLP export, and allowlisted cross-service correlation. Consumers must replace -removed version 1 imports before upgrading. Logging and OTel now have separate -shutdown timeout settings, shutdown failures are contained per subsystem, and -process recovery replaces inherited locks and asynchronous delivery state. -Google Cloud deployment manifests now render from validated repository -variables, with the Modal workspace identifier supplied as a protected secret. +spans, idempotent Flask and FastAPI adapters, explicit HTTPX and Python logging +instrumentation, provider-neutral logging destination strategies, independent +trace and metric OTLP exporters, pluggable exporter authentication, and +allowlisted cross-service correlation. Every remote logging destination uses +an independent bounded queue, and logging and OTel failures remain isolated +from application responses and calculations. Consumers must replace removed +version 1 imports and provide their own identity, destinations, attribute +policy, endpoints, and credentials before upgrading. diff --git a/deploy/gcp/README.md b/deploy/gcp/README.md deleted file mode 100644 index 108c605..0000000 --- a/deploy/gcp/README.md +++ /dev/null @@ -1,251 +0,0 @@ -# Google Cloud deployment plan - -This directory defines the centralized API v1 observability resources in the -project selected by `OBSERVABILITY_PROJECT_ID`. The workload boundary is -defined in -[`workload-inventory.template.yaml`](workload-inventory.template.yaml). -Applications absent from that inventory receive no credentials or destination -permissions. - -These files are a reviewable deployment plan. Applying them changes live IAM, -Cloud Logging routing, Cloud Run, and monitoring resources and therefore -requires an operator-approved deployment window. - -## Resources - -| File | Resource | -| --- | --- | -| `collector/config.yaml` | OTLP gRPC receiver, bounded processors, and Google Telemetry API exporter for traces and metrics | -| `collector/Dockerfile` | Google-built OTel Collector 0.160.0 plus the reviewed configuration | -| `collector/service.template.yaml` | Authenticated Cloud Run collector with fixed CPU, memory, concurrency, health checks, and scaling bounds | -| `log-routing.template.yaml` | Exact Cloud Run source sinks, restricted Modal direct-log sink, and `_Default` duplicate exclusion | -| `iam.template.yaml` | Collector and Modal service accounts, Cloud Run invokers, and a dedicated Modal API v1 identity provider | -| `dashboard.template.json` | Initial request, latency, error, dropped-item, and exporter-failure dashboard | -| `alerts.template.yaml` | Initial alert policy inputs | -| `render_deployment.py` | Validates deployment variables and writes private rendered files under `rendered/` | -| `verify.sh` | Read-only resource and routing checks after deployment | - -The collector accepts traces and metrics. Application logs do not enter the -collector. Cloud Run JSON output uses source-project sinks, while authorized -Modal processes use the package's bounded Cloud Logging writer. - -The initial collector tail policy retains 100% of traces. Separate error and -30-second latency policies are evaluated before the general policy. If the -general percentage is reduced after the volume review, those two policies keep -error and slow traces. SDK head sampling must remain at 100% for the collector -to receive spans needed for this decision. - -## Deployment order - -### 1. Configure and render deployment values - -Configure these GitHub Actions repository variables: - -- `OBSERVABILITY_PROJECT_ID` -- `OBSERVABILITY_PROJECT_NUMBER` -- `API_PROJECT_ID` -- `SIMULATION_ENTRY_PROJECT_ID` - -Configure `MODAL_WORKSPACE_ID` as a GitHub Actions repository secret. It is -private deployment metadata and must not be printed by workflows. - -Workflows that render or apply the deployment must map the values explicitly: - -```yaml -env: - OBSERVABILITY_PROJECT_ID: ${{ vars.OBSERVABILITY_PROJECT_ID }} - OBSERVABILITY_PROJECT_NUMBER: ${{ vars.OBSERVABILITY_PROJECT_NUMBER }} - API_PROJECT_ID: ${{ vars.API_PROJECT_ID }} - SIMULATION_ENTRY_PROJECT_ID: ${{ vars.SIMULATION_ENTRY_PROJECT_ID }} - MODAL_WORKSPACE_ID: ${{ secrets.MODAL_WORKSPACE_ID }} -``` - -For an operator-run deployment, set the same five values in the local process -without writing them to a tracked file, then render the templates: - -```bash -.venv/bin/python deploy/gcp/render_deployment.py -``` - -The renderer validates every value, reports only variable names, and writes -files with owner-only permissions under the ignored `deploy/gcp/rendered/` -directory. - -### 2. Enable services - -```bash -gcloud services enable \ - artifactregistry.googleapis.com \ - cloudbuild.googleapis.com \ - cloudresourcemanager.googleapis.com \ - iamcredentials.googleapis.com \ - logging.googleapis.com \ - monitoring.googleapis.com \ - run.googleapis.com \ - sts.googleapis.com \ - telemetry.googleapis.com \ - tracing.googleapis.com \ - --project="${OBSERVABILITY_PROJECT_ID}" -``` - -### 3. Create identities - -Create these service accounts in the central project: - -```text -policyengine-otel-collector@${OBSERVABILITY_PROJECT_ID}.iam.gserviceaccount.com -policyengine-api-v1-modal@${OBSERVABILITY_PROJECT_ID}.iam.gserviceaccount.com -``` - -Grant only the roles listed in the rendered `iam.yaml`. The collector receives -`roles/telemetry.writer` and `roles/serviceusage.serviceUsageConsumer`. The -Modal identity receives `roles/logging.logWriter` and collector invocation -permission. The four existing Cloud Run identities in the inventory receive -collector invocation permission on the collector service only. - -Remove project-level `roles/logging.logWriter` bindings from every identity -outside this inventory. Source-project logging service agents use conditional -`roles/logging.bucketWriter` access to the named analytics bucket and do not -receive project-level log write access. `verify.sh` fails when another -project-level log writer is present. - -Create a separate `modal-api-v1` workload identity pool and provider using the -issuer, audience, mappings, workspace, environment, and application condition -in the rendered `iam.yaml`. Do not modify the existing `modal/modal` provider -during this deployment; it belongs to applications excluded from this change. -Grant the new provider permission to impersonate only the API v1 Modal service -account. - -Before enabling the provider, decode one production and one staging Modal -identity token locally and confirm that `workspace_id`, `environment_name`, -and `app_name` exactly match the reviewed condition. - -### 4. Build and deploy the collector - -```bash -gcloud artifacts repositories create observability \ - --repository-format=docker \ - --location=us-central1 \ - --immutable-tags \ - --project="${OBSERVABILITY_PROJECT_ID}" - -gcloud builds submit deploy/gcp/collector \ - --tag="us-central1-docker.pkg.dev/${OBSERVABILITY_PROJECT_ID}/observability/otel-collector:0.160.0-api-v1-1" \ - --project="${OBSERVABILITY_PROJECT_ID}" - -gcloud run services replace deploy/gcp/rendered/collector/service.yaml \ - --region=us-central1 \ - --project="${OBSERVABILITY_PROJECT_ID}" -``` - -Apply `roles/run.invoker` bindings for the five identities listed in the -rendered `iam.yaml`. Do not grant unauthenticated invocation. Record the HTTPS -service URL as both `OTEL_EXPORTER_OTLP_ENDPOINT` and -`POLICYENGINE_OTEL_GOOGLE_AUDIENCE` in participating service configuration. - -### 5. Configure log routing - -Create one aggregated sink in each source project using the exact service and -schema filters in the rendered `log-routing.yaml`. Grant each generated sink -writer identity `roles/logging.bucketWriter` on the central log bucket. - -Update `policyengine-observability-app-logs` to the listed direct-log filter. -Add the listed exclusion to `_Default`; this prevents a direct Modal log from -being stored in both `_Default` and the analytics bucket. Preserve the existing -Cloud Audit Log exclusions. - -After routing one synthetic record per participating service, confirm each -`insertId` exists exactly once in the central project. - -### 6. Create dashboard and alerts - -```bash -gcloud monitoring dashboards create \ - --config-from-file=deploy/gcp/rendered/dashboard.json \ - --project="${OBSERVABILITY_PROJECT_ID}" -``` - -Create the API-ready alert policies with: - -```bash -.venv/bin/python deploy/gcp/create_alerts.py -``` - -The script is idempotent by policy display name. It leaves notification-channel -configuration empty when the project has no channel; add operator-owned channel -identifiers after creating the relevant email, Slack, or paging destination. - -### 7. Verify before consumer deployment - -```bash -bash deploy/gcp/verify.sh -``` - -Then use an approved workload identity to send one trace and metric. Attempt -the same request with a synthetic Modal token whose application name is not in -the inventory; token exchange or collector invocation must return permission -denial. Do not invoke an excluded application to perform this check. - -For an operator-run Cloud Run identity check, temporarily grant the operator -`roles/iam.serviceAccountTokenCreator` on one inventoried runtime identity, run: - -```bash -.venv/bin/python deploy/gcp/verify_otel.py \ - --endpoint="${POLICYENGINE_OTEL_GOOGLE_AUDIENCE}" \ - --service-account="sim-entry-beta-runtime@${SIMULATION_ENTRY_PROJECT_ID}.iam.gserviceaccount.com" -``` - -Remove the temporary operator binding immediately after the check. The script -requires an authenticated `gcloud` session, sends one trace and metric, verifies -both Google Cloud stores, and confirms that the collector rejects OTLP logs. - -Use [`verify_modal_wif.py`](verify_modal_wif.py) with the Modal CLI to run an -allowed app name and a synthetic denied app name. The remote function exchanges -its automatically injected Modal OIDC token, verifies service-account access, -invokes the collector, and writes one routing record without exposing any -token. Run an allowed app name from a temporary non-allowlisted environment to -verify the environment restriction, then delete that environment. - -## Rollback - -1. Remove the OTel endpoint from participating service configuration. Local - structured logging continues and no remote OTel exporter is created. -2. Remove Modal remote logging configuration. Modal JSON output continues. -3. Revert each participating service to its previous package version and - deployment revision. -4. Remove the new source sinks and restore the prior central direct-log sink - filter and `_Default` exclusion state. -5. Remove invoker bindings, disable the `modal-api-v1` provider, and disable or - delete the collector service. -6. Keep the central bucket during the retention period unless the stored data - itself caused the incident. - -Rollback does not modify the existing `modal/modal` provider or any excluded -application deployment. - -## Deployment record - -The infrastructure portion of this runbook was applied and verified on -2026-09-22: - -- the global central log bucket retains records for 30 - days and has log analytics enabled; -- exact source-project sinks route the two API services and the two simulation - entry services to that bucket; -- the authenticated collector runs in `us-central1` as - `policyengine-api-v1-otel-collector`; -- the dedicated `modal-api-v1` identity provider is active with the workspace, - environment, and application conditions in the rendered `iam.yaml`; -- the only project-level `roles/logging.logWriter` identity is the API v1 - Modal service account; -- the dashboard and six alert policies are present and enabled; and -- the project currently has no alert notification channel, so the policies - record incidents without sending email, Slack, or paging notifications. - -The package and consumer service rollout remains pending until the three draft -pull requests are reviewed, the package is published as version 2.0, and the -temporary Git source pins in both consumer repositories are replaced with the -published version. Run the synthetic cross-service request, volume and cost -measurement, and destination comparison after those deployments. Record the -deployed revisions and the observation interval here before declaring the -consumer rollout complete. diff --git a/deploy/gcp/alerts.template.yaml b/deploy/gcp/alerts.template.yaml deleted file mode 100644 index 19fdbac..0000000 --- a/deploy/gcp/alerts.template.yaml +++ /dev/null @@ -1,42 +0,0 @@ -policies: - - display_name: API v1 exporter failures - condition_type: prometheus_query_language - query: 'sum(increase({"policyengine.telemetry.exporter.failure"}[5m])) > 0' - duration: 0s - evaluation_interval: 60s - disable_metric_validation: true - - display_name: API v1 dropped telemetry - condition_type: prometheus_query_language - query: 'sum(increase({"policyengine.telemetry.dropped"}[5m])) > 0' - duration: 0s - evaluation_interval: 60s - disable_metric_validation: true - - display_name: API v1 elevated error rate - condition_type: prometheus_query_language - query: 'sum(increase({"policyengine.error.count"}[10m])) > 1' - duration: 0s - evaluation_interval: 60s - disable_metric_validation: true - - display_name: API v1 high request latency - condition_type: prometheus_query_language - query: >- - histogram_quantile(0.99, sum by (le) - (rate({"policyengine.request.duration_bucket"}[5m]))) > 30 - duration: 300s - evaluation_interval: 60s - disable_metric_validation: true - - display_name: API v1 collector unavailable - condition_type: metric_threshold - metric_type: run.googleapis.com/service_health_count - resource_label_service_name: policyengine-api-v1-otel-collector - metric_label_service_health: UNHEALTHY - comparison: COMPARISON_GT - threshold: 0 - duration: 300s - - display_name: API v1 monthly log ingestion above 10 GiB - condition_type: metric_threshold - metric_type: logging.googleapis.com/billing/log_bucket_monthly_bytes_ingested - metric_label_log_bucket_id: ${OBSERVABILITY_PROJECT_ID} - comparison: COMPARISON_GT - threshold: 10737418240 - duration: 0s diff --git a/deploy/gcp/collector/Dockerfile b/deploy/gcp/collector/Dockerfile deleted file mode 100644 index ae3f632..0000000 --- a/deploy/gcp/collector/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM us-docker.pkg.dev/cloud-ops-agents-artifacts/google-cloud-opentelemetry-collector/otelcol-google:0.160.0 - -COPY config.yaml /etc/otelcol-google/config.yaml - -ENTRYPOINT ["/otelcol-google"] -CMD ["--config=/etc/otelcol-google/config.yaml"] diff --git a/deploy/gcp/collector/config.yaml b/deploy/gcp/collector/config.yaml deleted file mode 100644 index 2fa3ced..0000000 --- a/deploy/gcp/collector/config.yaml +++ /dev/null @@ -1,71 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:8080 - -processors: - memory_limiter: - check_interval: 1s - limit_mib: 384 - spike_limit_mib: 64 - resource/destination: - attributes: - - key: gcp.project_id - value: ${env:OBSERVABILITY_PROJECT_ID} - action: upsert - tail_sampling: - decision_wait: 30s - num_traces: 50000 - expected_new_traces_per_sec: 500 - policies: - - name: retain-errors - type: status_code - status_code: - status_codes: [ERROR] - - name: retain-slow-operations - type: latency - latency: - threshold_ms: 30000 - - name: initial-full-sample - type: probabilistic - probabilistic: - sampling_percentage: 100 - batch: - send_batch_size: 200 - send_batch_max_size: 1000 - timeout: 5s - -exporters: - otlp_grpc: - endpoint: telemetry.googleapis.com:443 - auth: - authenticator: googleclientauth - sending_queue: - enabled: true - queue_size: 2000 - retry_on_failure: - enabled: true - initial_interval: 1s - max_interval: 5s - max_elapsed_time: 30s - -extensions: - googleclientauth: - health_check: - endpoint: 0.0.0.0:13133 - -service: - extensions: [googleclientauth, health_check] - pipelines: - traces: - receivers: [otlp] - processors: [memory_limiter, resource/destination, tail_sampling, batch] - exporters: [otlp_grpc] - metrics: - receivers: [otlp] - processors: [memory_limiter, resource/destination, batch] - exporters: [otlp_grpc] - telemetry: - logs: - level: info diff --git a/deploy/gcp/collector/service.template.yaml b/deploy/gcp/collector/service.template.yaml deleted file mode 100644 index 8cde2e4..0000000 --- a/deploy/gcp/collector/service.template.yaml +++ /dev/null @@ -1,48 +0,0 @@ -apiVersion: serving.knative.dev/v1 -kind: Service -metadata: - name: policyengine-api-v1-otel-collector - namespace: "${OBSERVABILITY_PROJECT_NUMBER}" - labels: - cloud.googleapis.com/location: us-central1 - annotations: - run.googleapis.com/ingress: all -spec: - template: - metadata: - annotations: - autoscaling.knative.dev/minScale: "1" - autoscaling.knative.dev/maxScale: "10" - run.googleapis.com/cpu-throttling: "false" - run.googleapis.com/startup-cpu-boost: "true" - spec: - containerConcurrency: 100 - timeoutSeconds: 30 - serviceAccountName: policyengine-otel-collector@${OBSERVABILITY_PROJECT_ID}.iam.gserviceaccount.com - containers: - - name: collector - image: us-central1-docker.pkg.dev/${OBSERVABILITY_PROJECT_ID}/observability/otel-collector:0.160.0-api-v1-1 - env: - - name: OBSERVABILITY_PROJECT_ID - value: ${OBSERVABILITY_PROJECT_ID} - ports: - - name: h2c - containerPort: 8080 - resources: - limits: - cpu: "1" - memory: 512Mi - startupProbe: - httpGet: - path: / - port: 13133 - periodSeconds: 2 - timeoutSeconds: 1 - failureThreshold: 30 - livenessProbe: - httpGet: - path: / - port: 13133 - periodSeconds: 30 - timeoutSeconds: 2 - failureThreshold: 3 diff --git a/deploy/gcp/create_alerts.py b/deploy/gcp/create_alerts.py deleted file mode 100644 index f31df88..0000000 --- a/deploy/gcp/create_alerts.py +++ /dev/null @@ -1,255 +0,0 @@ -"""Create the initial API v1 Cloud Monitoring alert policies idempotently.""" - -from __future__ import annotations - -import json -import os -import subprocess -import urllib.parse -import urllib.request -from typing import Any - -PROJECT = os.environ.get("OBSERVABILITY_PROJECT_ID", "") -API_ROOT = f"https://monitoring.googleapis.com/v3/projects/{PROJECT}" - - -def _threshold_condition( - *, - display_name: str, - filter_value: str, - comparison: str, - threshold: float, - duration: str, - alignment_period: str, - aligner: str, - reducer: str, -) -> dict[str, Any]: - return { - "displayName": display_name, - "conditionThreshold": { - "filter": filter_value, - "comparison": comparison, - "thresholdValue": threshold, - "duration": duration, - "aggregations": [ - { - "alignmentPeriod": alignment_period, - "perSeriesAligner": aligner, - "crossSeriesReducer": reducer, - } - ], - "trigger": {"count": 1}, - }, - } - - -def _promql_condition( - *, - display_name: str, - query: str, - duration: str, - alert_rule: str, -) -> dict[str, Any]: - return { - "displayName": display_name, - "conditionPrometheusQueryLanguage": { - "query": query, - "duration": duration, - "evaluationInterval": "60s", - "alertRule": alert_rule, - "ruleGroup": "policyengine_api_v1", - "disableMetricValidation": True, - }, - } - - -def _policy( - display_name: str, - condition: dict[str, Any], - documentation: str, -) -> dict[str, Any]: - return { - "displayName": display_name, - "combiner": "OR", - "enabled": True, - "notificationChannels": [], - "documentation": { - "content": documentation, - "mimeType": "text/markdown", - }, - "alertStrategy": {"autoClose": "1800s"}, - "conditions": [condition], - } - - -POLICIES = [ - _policy( - "API v1 exporter failures", - _promql_condition( - display_name="Exporter failure counter increased", - query=( - 'sum(increase({"policyengine.telemetry.exporter.failure"}' - "[5m])) > 0" - ), - duration="0s", - alert_rule="ExporterFailure", - ), - "The API v1 runtime reported at least one telemetry exporter failure.", - ), - _policy( - "API v1 dropped telemetry", - _promql_condition( - display_name="Dropped telemetry counter increased", - query=( - 'sum(increase({"policyengine.telemetry.dropped"}[5m])) > 0' - ), - duration="0s", - alert_rule="DroppedTelemetry", - ), - "An API v1 bounded telemetry queue dropped at least one item.", - ), - _policy( - "API v1 elevated error rate", - _promql_condition( - display_name="More than one application error in ten minutes", - query='sum(increase({"policyengine.error.count"}[10m])) > 1', - duration="0s", - alert_rule="ElevatedErrorRate", - ), - "API v1 application error counters increased more than once in ten minutes.", - ), - _policy( - "API v1 high request latency", - _promql_condition( - display_name="P99 request duration above 30 seconds", - query=( - "histogram_quantile(0.99, sum by (le) " - '(rate({"policyengine.request.duration_bucket"}[5m]))) > 30' - ), - duration="300s", - alert_rule="HighRequestLatency", - ), - "API v1 P99 request duration exceeded 30 seconds for five minutes.", - ), - _policy( - "API v1 collector unavailable", - _threshold_condition( - display_name="Collector Cloud Run service is unhealthy", - filter_value=( - 'resource.type="cloud_run_revision" AND ' - 'metric.type="run.googleapis.com/service_health_count" AND ' - 'resource.label."service_name"=' - '"policyengine-api-v1-otel-collector" AND ' - 'metric.label."service_health"="UNHEALTHY"' - ), - comparison="COMPARISON_GT", - threshold=0, - duration="300s", - alignment_period="60s", - aligner="ALIGN_MEAN", - reducer="REDUCE_MAX", - ), - "The authenticated API v1 collector reported an unhealthy revision.", - ), - _policy( - "API v1 monthly log ingestion above 10 GiB", - _threshold_condition( - display_name="Central analytics bucket exceeds 10 GiB this month", - filter_value=( - 'resource.type="global" AND ' - 'metric.type="logging.googleapis.com/billing/' - 'log_bucket_monthly_bytes_ingested" AND ' - f'metric.label."log_bucket_id"="{PROJECT}"' - ), - comparison="COMPARISON_GT", - threshold=10_737_418_240, - duration="0s", - alignment_period="1800s", - aligner="ALIGN_MAX", - reducer="REDUCE_SUM", - ), - "The central API v1 analytics bucket exceeded 10 GiB of month-to-date ingestion.", - ), -] - - -def _token() -> str: - return subprocess.check_output( - ["gcloud", "auth", "print-access-token"], - text=True, - ).strip() - - -def _request( - url: str, - *, - token: str, - payload: dict[str, Any] | None = None, -) -> dict[str, Any]: - request = urllib.request.Request( - url, - data=json.dumps(payload).encode() if payload is not None else None, - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - }, - method="POST" if payload is not None else "GET", - ) - with urllib.request.urlopen(request, timeout=30) as response: - return json.loads(response.read()) - - -def main() -> None: - if not PROJECT: - raise SystemExit( - "Missing deployment variable: OBSERVABILITY_PROJECT_ID" - ) - token = _token() - existing: dict[str, str] = {} - page_token = "" - while True: - query = ( - urllib.parse.urlencode({"pageToken": page_token}) - if page_token - else "" - ) - suffix = f"?{query}" if query else "" - response = _request( - f"{API_ROOT}/alertPolicies{suffix}", - token=token, - ) - for policy in response.get("alertPolicies", []): - existing[str(policy["displayName"])] = str(policy["name"]) - page_token = str(response.get("nextPageToken", "")) - if not page_token: - break - - results = [] - for policy in POLICIES: - display_name = str(policy["displayName"]) - if display_name in existing: - results.append( - { - "displayName": display_name, - "name": existing[display_name], - "status": "existing", - } - ) - continue - created = _request( - f"{API_ROOT}/alertPolicies", - token=token, - payload=policy, - ) - results.append( - { - "displayName": display_name, - "name": created["name"], - "status": "created", - } - ) - print(json.dumps(results, indent=2, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/deploy/gcp/dashboard.template.json b/deploy/gcp/dashboard.template.json deleted file mode 100644 index 2fa07b2..0000000 --- a/deploy/gcp/dashboard.template.json +++ /dev/null @@ -1,183 +0,0 @@ -{ - "displayName": "PolicyEngine API v1 observability", - "mosaicLayout": { - "columns": 12, - "tiles": [ - { - "xPos": 0, - "yPos": 0, - "width": 6, - "height": 4, - "widget": { - "title": "Request rate by service", - "xyChart": { - "dataSets": [{ - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"prometheus.googleapis.com/policyengine.request.count/counter\"", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_RATE", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": ["resource.label.\"job\""] - } - } - }, - "plotType": "LINE" - }] - } - } - }, - { - "xPos": 6, - "yPos": 0, - "width": 6, - "height": 4, - "widget": { - "title": "Request duration", - "xyChart": { - "dataSets": [{ - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"prometheus.googleapis.com/policyengine.request.duration/histogram\"", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_PERCENTILE_99" - } - } - }, - "plotType": "LINE" - }] - } - } - }, - { - "xPos": 0, - "yPos": 4, - "width": 6, - "height": 4, - "widget": { - "title": "Application errors", - "xyChart": { - "dataSets": [{ - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"prometheus.googleapis.com/policyengine.error.count/counter\"", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_RATE", - "crossSeriesReducer": "REDUCE_SUM" - } - } - }, - "plotType": "STACKED_AREA" - }] - } - } - }, - { - "xPos": 6, - "yPos": 4, - "width": 6, - "height": 4, - "widget": { - "title": "Dropped telemetry and exporter failures", - "xyChart": { - "dataSets": [ - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"prometheus.googleapis.com/policyengine.telemetry.dropped/counter\"", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_RATE", - "crossSeriesReducer": "REDUCE_SUM" - } - } - }, - "plotType": "LINE" - }, - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "metric.type=\"prometheus.googleapis.com/policyengine.telemetry.exporter.failure/counter\"", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_RATE", - "crossSeriesReducer": "REDUCE_SUM" - } - } - }, - "plotType": "LINE" - } - ] - } - } - }, - { - "xPos": 0, - "yPos": 8, - "width": 6, - "height": 4, - "widget": { - "title": "Collector health", - "xyChart": { - "dataSets": [{ - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "resource.type=\"cloud_run_revision\" AND metric.type=\"run.googleapis.com/service_health_count\" AND resource.label.\"service_name\"=\"policyengine-api-v1-otel-collector\"", - "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN", - "crossSeriesReducer": "REDUCE_SUM", - "groupByFields": ["metric.label.\"service_health\""] - } - } - }, - "plotType": "LINE" - }] - } - } - }, - { - "xPos": 6, - "yPos": 8, - "width": 6, - "height": 4, - "widget": { - "title": "Telemetry ingestion volume", - "xyChart": { - "dataSets": [ - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "resource.type=\"global\" AND metric.type=\"logging.googleapis.com/billing/log_bucket_bytes_ingested\" AND metric.label.\"log_bucket_id\"=\"${OBSERVABILITY_PROJECT_ID}\"", - "aggregation": { - "alignmentPeriod": "3600s", - "perSeriesAligner": "ALIGN_SUM", - "crossSeriesReducer": "REDUCE_SUM" - } - } - }, - "plotType": "LINE" - }, - { - "timeSeriesQuery": { - "timeSeriesFilter": { - "filter": "resource.type=\"global\" AND metric.type=\"monitoring.googleapis.com/billing/samples_ingested\" AND metric.label.\"metric_domain\"=\"prometheus.googleapis.com\"", - "aggregation": { - "alignmentPeriod": "3600s", - "perSeriesAligner": "ALIGN_SUM", - "crossSeriesReducer": "REDUCE_SUM" - } - } - }, - "plotType": "LINE" - } - ] - } - } - } - ] - } -} diff --git a/deploy/gcp/iam.template.yaml b/deploy/gcp/iam.template.yaml deleted file mode 100644 index 062e90b..0000000 --- a/deploy/gcp/iam.template.yaml +++ /dev/null @@ -1,36 +0,0 @@ -service_accounts: - collector: - email: policyengine-otel-collector@${OBSERVABILITY_PROJECT_ID}.iam.gserviceaccount.com - project_roles: - - roles/telemetry.writer - - roles/serviceusage.serviceUsageConsumer - modal: - email: policyengine-api-v1-modal@${OBSERVABILITY_PROJECT_ID}.iam.gserviceaccount.com - project_roles: - - roles/logging.logWriter - collector_roles: - - roles/run.invoker - -collector_invokers: - - policyengine-api-cr-runtime@${API_PROJECT_ID}.iam.gserviceaccount.com - - policyengine-api-cr-staging@${API_PROJECT_ID}.iam.gserviceaccount.com - - sim-entry-prod-runtime@${SIMULATION_ENTRY_PROJECT_ID}.iam.gserviceaccount.com - - sim-entry-beta-runtime@${SIMULATION_ENTRY_PROJECT_ID}.iam.gserviceaccount.com - - policyengine-api-v1-modal@${OBSERVABILITY_PROJECT_ID}.iam.gserviceaccount.com - -modal_workload_identity: - pool_id: modal-api-v1 - provider_id: modal-api-v1 - issuer: https://oidc.modal.com - allowed_audience: oidc.modal.com - workspace_id: ${MODAL_WORKSPACE_ID} - attribute_mapping: - google.subject: assertion.app_name - attribute.app_name: assertion.app_name - attribute.environment_name: assertion.environment_name - attribute.workspace_id: assertion.workspace_id - attribute_condition: >- - assertion.workspace_id == "${MODAL_WORKSPACE_ID}" && - (assertion.environment_name == "main" || assertion.environment_name == "staging") && - (assertion.app_name == "policyengine-simulation-gateway" || - assertion.app_name.matches("^policyengine-simulation-py[0-9]+-[0-9]+-[0-9]+$")) diff --git a/deploy/gcp/log-routing.template.yaml b/deploy/gcp/log-routing.template.yaml deleted file mode 100644 index 8d84ed8..0000000 --- a/deploy/gcp/log-routing.template.yaml +++ /dev/null @@ -1,37 +0,0 @@ -destination: - project: ${OBSERVABILITY_PROJECT_ID} - location: global - bucket: ${OBSERVABILITY_PROJECT_ID} - direct_log_name: policyengine-api-v1-modal - -source_sinks: - - project: ${API_PROJECT_ID} - name: api-v1-central-observability - filter: >- - resource.type="cloud_run_revision" AND - (resource.labels.service_name="policyengine-api" OR - resource.labels.service_name="policyengine-api-staging") AND - jsonPayload.schema_version="policyengine.observability.v2" - - project: ${SIMULATION_ENTRY_PROJECT_ID} - name: api-v1-central-observability - filter: >- - resource.type="cloud_run_revision" AND - (resource.labels.service_name="policyengine-simulation-entry" OR - resource.labels.service_name="policyengine-simulation-entry-staging") AND - jsonPayload.schema_version="policyengine.observability.v2" - -central_direct_sink: - name: policyengine-observability-app-logs - filter: >- - LOG_ID("policyengine-api-v1-modal") AND - jsonPayload.schema_version="policyengine.observability.v2" AND - jsonPayload."service.namespace"="policyengine.api-v1" - -default_bucket_exclusion: - name: exclude-api-v1-direct-observability - filter: >- - LOG_ID("policyengine-api-v1-modal") AND - jsonPayload.schema_version="policyengine.observability.v2" AND - jsonPayload."service.namespace"="policyengine.api-v1" AND - (jsonPayload."service.name"="policyengine-simulation-gateway" OR - jsonPayload."service.name"=~"^policyengine-simulation-py[0-9]+-[0-9]+-[0-9]+$") diff --git a/deploy/gcp/render_deployment.py b/deploy/gcp/render_deployment.py deleted file mode 100644 index b85613e..0000000 --- a/deploy/gcp/render_deployment.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Render Google Cloud deployment templates from validated environment values.""" - -from __future__ import annotations - -import argparse -import os -import re -from pathlib import Path - -ROOT = Path(__file__).resolve().parent -DEFAULT_OUTPUT = ROOT / "rendered" -PLACEHOLDER = re.compile(r"\$\{([A-Z][A-Z0-9_]*)\}") -TEMPLATES = ( - ("iam.template.yaml", "iam.yaml"), - ("workload-inventory.template.yaml", "workload-inventory.yaml"), - ("log-routing.template.yaml", "log-routing.yaml"), - ("alerts.template.yaml", "alerts.yaml"), - ("dashboard.template.json", "dashboard.json"), - ("collector/service.template.yaml", "collector/service.yaml"), -) -VALIDATORS = { - "OBSERVABILITY_PROJECT_ID": re.compile(r"[a-z][a-z0-9-]{4,28}[a-z0-9]"), - "OBSERVABILITY_PROJECT_NUMBER": re.compile(r"[1-9][0-9]{5,29}"), - "API_PROJECT_ID": re.compile(r"[a-z][a-z0-9-]{4,28}[a-z0-9]"), - "SIMULATION_ENTRY_PROJECT_ID": re.compile(r"[a-z][a-z0-9-]{4,28}[a-z0-9]"), - "MODAL_WORKSPACE_ID": re.compile(r"ac-[A-Za-z0-9_-]+"), -} - - -def _deployment_values() -> dict[str, str]: - missing = sorted(name for name in VALIDATORS if not os.getenv(name)) - if missing: - raise SystemExit("Missing deployment variables: " + ", ".join(missing)) - - values = {name: os.environ[name] for name in VALIDATORS} - invalid = sorted( - name - for name, pattern in VALIDATORS.items() - if pattern.fullmatch(values[name]) is None - ) - if invalid: - raise SystemExit("Invalid deployment variables: " + ", ".join(invalid)) - return values - - -def _render(content: str, values: dict[str, str]) -> str: - referenced = set(PLACEHOLDER.findall(content)) - unknown = sorted(referenced - values.keys()) - if unknown: - raise SystemExit("Unknown deployment variables: " + ", ".join(unknown)) - rendered = PLACEHOLDER.sub(lambda match: values[match.group(1)], content) - unresolved = sorted(set(PLACEHOLDER.findall(rendered))) - if unresolved: - raise SystemExit( - "Unresolved deployment variables: " + ", ".join(unresolved) - ) - return rendered - - -def render_deployment(output_directory: Path) -> None: - values = _deployment_values() - rendered_files = [] - for source_name, output_name in TEMPLATES: - source = ROOT / source_name - rendered_files.append( - (output_name, _render(source.read_text(), values)) - ) - - for output_name, content in rendered_files: - destination = output_directory / output_name - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(content) - destination.chmod(0o600) - - variable_names = ", ".join(sorted(values)) - print( - f"Rendered {len(TEMPLATES)} deployment files using: {variable_names}" - ) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument( - "--output-dir", - type=Path, - default=DEFAULT_OUTPUT, - help="Destination for rendered files (default: deploy/gcp/rendered)", - ) - arguments = parser.parse_args() - render_deployment(arguments.output_dir) - - -if __name__ == "__main__": - main() diff --git a/deploy/gcp/verify.sh b/deploy/gcp/verify.sh deleted file mode 100644 index a14188a..0000000 --- a/deploy/gcp/verify.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -: "${OBSERVABILITY_PROJECT_ID:?Missing OBSERVABILITY_PROJECT_ID}" -: "${API_PROJECT_ID:?Missing API_PROJECT_ID}" -: "${SIMULATION_ENTRY_PROJECT_ID:?Missing SIMULATION_ENTRY_PROJECT_ID}" - -project="${OBSERVABILITY_PROJECT_ID}" -region="us-central1" -collector="policyengine-api-v1-otel-collector" - -gcloud logging buckets describe "${OBSERVABILITY_PROJECT_ID}" \ - --location=global \ - --project="${project}" \ - --format='value(name,retentionDays,analyticsEnabled)' - -gcloud run services describe "${collector}" \ - --region="${region}" \ - --project="${project}" \ - --format='value(status.url,spec.template.spec.serviceAccountName)' - -gcloud projects get-iam-policy "${project}" \ - --flatten='bindings[].members' \ - --filter='bindings.role:roles/telemetry.writer OR bindings.role:roles/serviceusage.serviceUsageConsumer OR bindings.role:roles/logging.logWriter' \ - --format='table(bindings.role,bindings.members)' - -log_writers="$( - gcloud projects get-iam-policy "${project}" \ - --flatten='bindings[].members' \ - --filter='bindings.role=roles/logging.logWriter' \ - --format='value(bindings.members)' -)" -expected_log_writer="serviceAccount:policyengine-api-v1-modal@${OBSERVABILITY_PROJECT_ID}.iam.gserviceaccount.com" -if [[ "${log_writers}" != "${expected_log_writer}" ]]; then - echo "Unexpected project-level Cloud Logging writers: ${log_writers}" >&2 - exit 1 -fi - -gcloud logging views get-iam-policy _AllLogs \ - --bucket="${OBSERVABILITY_PROJECT_ID}" \ - --location=global \ - --project="${project}" \ - --format=json - -gcloud iam workload-identity-pools providers describe modal-api-v1 \ - --workload-identity-pool=modal-api-v1 \ - --location=global \ - --project="${project}" \ - --format='yaml(state,attributeCondition,attributeMapping,oidc)' - -for source_project in "${API_PROJECT_ID}" "${SIMULATION_ENTRY_PROJECT_ID}"; do - gcloud logging sinks describe api-v1-central-observability \ - --project="${source_project}" \ - --format='yaml(destination,filter,writerIdentity)' -done - -gcloud logging sinks describe policyengine-observability-app-logs \ - --project="${project}" \ - --format='yaml(destination,filter)' - -gcloud logging sinks describe _Default \ - --project="${project}" \ - --format='yaml(filter,exclusions)' diff --git a/deploy/gcp/verify_modal_wif.py b/deploy/gcp/verify_modal_wif.py deleted file mode 100644 index 7c8aaaa..0000000 --- a/deploy/gcp/verify_modal_wif.py +++ /dev/null @@ -1,224 +0,0 @@ -"""Verify allowed and denied Modal identities against Google WIF. - -Set ``POLICYENGINE_WIF_TEST_APP`` in the local process before ``modal run``. -The script never returns or prints the Modal identity token or Google tokens. -""" - -# pyright: reportMissingImports=false - -from __future__ import annotations - -import base64 -import hashlib -import json -import os -import time -import urllib.error -import urllib.parse -import urllib.request - -import modal - -POOL = "modal-api-v1" -PROVIDER = "modal-api-v1" -APP_NAME = os.environ.get( - "POLICYENGINE_WIF_TEST_APP", - "policyengine-observability-wif-denied-test", -) - -app = modal.App(APP_NAME) - - -def _required_environment(name: str) -> str: - value = os.getenv(name) - if not value: - raise SystemExit(f"Missing deployment variable: {name}") - return value - - -def _jwt_claims(token: str) -> dict[str, object]: - payload = token.split(".")[1] - payload += "=" * (-len(payload) % 4) - return json.loads(base64.urlsafe_b64decode(payload)) - - -def _post_form(url: str, values: dict[str, str]) -> dict[str, object]: - request = urllib.request.Request( - url, - data=urllib.parse.urlencode(values).encode(), - headers={"Content-Type": "application/x-www-form-urlencoded"}, - ) - with urllib.request.urlopen(request, timeout=15) as response: - return json.loads(response.read()) - - -def _post_json( - url: str, - payload: dict[str, object], - *, - bearer_token: str, -) -> tuple[int, dict[str, object]]: - request = urllib.request.Request( - url, - data=json.dumps(payload).encode(), - headers={ - "Authorization": f"Bearer {bearer_token}", - "Content-Type": "application/json", - }, - ) - with urllib.request.urlopen(request, timeout=15) as response: - body = response.read() - return response.status, json.loads(body) if body else {} - - -def _error_result(error: urllib.error.HTTPError) -> dict[str, object]: - try: - payload = json.loads(error.read()) - except (json.JSONDecodeError, UnicodeDecodeError): - payload = {} - error_payload = payload.get("error", {}) - if isinstance(error_payload, dict): - error_name = error_payload.get("status") or error_payload.get("error") - else: - error_name = error_payload - return { - "http_status": error.code, - "error": error_name, - } - - -@app.function(timeout=60) -def verify_identity( - project: str, - project_number: str, - workspace_id_digest: str, -) -> dict[str, object]: - service_account = ( - f"policyengine-api-v1-modal@{project}.iam.gserviceaccount.com" - ) - collector = ( - "https://policyengine-api-v1-otel-collector-" - f"{project_number}.us-central1.run.app" - ) - identity_token = os.environ["MODAL_IDENTITY_TOKEN"] - claims = _jwt_claims(identity_token) - safe_claims = { - key: claims.get(key) - for key in ( - "environment_name", - "app_name", - "function_name", - "aud", - "iss", - ) - } - workspace_id = str(claims.get("workspace_id", "")) - workspace_id_matches = ( - hashlib.sha256(workspace_id.encode()).hexdigest() - == workspace_id_digest - ) - audience = ( - "//iam.googleapis.com/projects/" - f"{project_number}/locations/global/workloadIdentityPools/{POOL}" - f"/providers/{PROVIDER}" - ) - try: - sts_payload = _post_form( - "https://sts.googleapis.com/v1/token", - { - "audience": audience, - "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", - "requested_token_type": ( - "urn:ietf:params:oauth:token-type:access_token" - ), - "scope": "https://www.googleapis.com/auth/cloud-platform", - "subject_token": identity_token, - "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", - }, - ) - except urllib.error.HTTPError as error: - return { - "claims": safe_claims, - "workspace_id_matches": workspace_id_matches, - "token_exchange": _error_result(error), - } - - federated_token = str(sts_payload["access_token"]) - service_account_url = ( - "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/" - f"{service_account}" - ) - access_status, access_payload = _post_json( - f"{service_account_url}:generateAccessToken", - { - "scope": ["https://www.googleapis.com/auth/cloud-platform"], - "lifetime": "300s", - }, - bearer_token=federated_token, - ) - service_access_token = str(access_payload["accessToken"]) - - identity_status, identity_payload = _post_json( - f"{service_account_url}:generateIdToken", - {"audience": collector, "includeEmail": True}, - bearer_token=federated_token, - ) - collector_identity_token = str(identity_payload["token"]) - collector_request = urllib.request.Request( - collector, - headers={"Authorization": f"Bearer {collector_identity_token}"}, - ) - try: - with urllib.request.urlopen(collector_request, timeout=15) as response: - collector_status = response.status - except urllib.error.HTTPError as error: - collector_status = error.code - - verification_id = f"modal-wif-{int(time.time())}" - logging_status, _ = _post_json( - "https://logging.googleapis.com/v2/entries:write", - { - "logName": f"projects/{project}/logs/policyengine-api-v1-modal", - "resource": { - "type": "global", - "labels": {"project_id": project}, - }, - "entries": [ - { - "insertId": verification_id, - "jsonPayload": { - "schema_version": "policyengine.observability.v2", - "verification_id": verification_id, - "service.namespace": "policyengine.api-v1", - "service.name": safe_claims["app_name"], - "message": "Modal WIF verification", - }, - } - ], - }, - bearer_token=service_access_token, - ) - return { - "claims": safe_claims, - "workspace_id_matches": workspace_id_matches, - "token_exchange": {"http_status": 200}, - "service_account_access": {"http_status": access_status}, - "service_account_identity": {"http_status": identity_status}, - "collector_http_status": collector_status, - "logging_http_status": logging_status, - "verification_id": verification_id, - } - - -@app.local_entrypoint() -def main() -> None: - project = _required_environment("OBSERVABILITY_PROJECT_ID") - project_number = _required_environment("OBSERVABILITY_PROJECT_NUMBER") - workspace_id = _required_environment("MODAL_WORKSPACE_ID") - workspace_id_digest = hashlib.sha256(workspace_id.encode()).hexdigest() - result = getattr(verify_identity, "remote")( - project, - project_number, - workspace_id_digest, - ) - print(json.dumps(result, sort_keys=True)) diff --git a/deploy/gcp/verify_otel.py b/deploy/gcp/verify_otel.py deleted file mode 100644 index 08a9e8a..0000000 --- a/deploy/gcp/verify_otel.py +++ /dev/null @@ -1,258 +0,0 @@ -"""Send and verify synthetic OTLP signals through the Cloud Run collector.""" - -from __future__ import annotations - -import argparse -import json -import os -import secrets -import subprocess -import time -import urllib.error -import urllib.parse -import urllib.request - -import grpc -from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( - OTLPMetricExporter, -) -from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( - OTLPSpanExporter, -) -from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ( - ExportLogsServiceRequest, -) -from opentelemetry.proto.collector.logs.v1.logs_service_pb2_grpc import ( - LogsServiceStub, -) -from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue -from opentelemetry.proto.logs.v1.logs_pb2 import ( - LogRecord, - ResourceLogs, - ScopeLogs, -) -from opentelemetry.proto.resource.v1.resource_pb2 import ( - Resource as ProtoResource, -) -from opentelemetry.sdk.metrics import MeterProvider -from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader -from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor - -PROJECT = os.environ.get("OBSERVABILITY_PROJECT_ID", "") -METRIC_NAME = "policyengine.verification.counter" -METRIC_TYPE = f"prometheus.googleapis.com/{METRIC_NAME}/counter" - - -def _gcloud_output(*arguments: str) -> str: - return subprocess.check_output( - ["gcloud", *arguments], - text=True, - ).strip() - - -def _identity_token(service_account: str, audience: str) -> str: - return _gcloud_output( - "auth", - "print-identity-token", - f"--impersonate-service-account={service_account}", - f"--audiences={audience}", - ) - - -def _access_token() -> str: - return _gcloud_output("auth", "print-access-token") - - -def _authorized_get(url: str, token: str) -> dict[str, object]: - request = urllib.request.Request( - url, - headers={"Authorization": f"Bearer {token}"}, - ) - with urllib.request.urlopen(request, timeout=15) as response: - return json.loads(response.read()) - - -def _send_signals(endpoint: str, service_account: str) -> dict[str, object]: - identity_token = _identity_token(service_account, endpoint) - headers = (("authorization", f"Bearer {identity_token}"),) - credentials = grpc.ssl_channel_credentials() - resource = Resource.create( - { - "service.name": "policyengine-observability-verification", - "service.namespace": "policyengine.api-v1", - "service.version": "2.0.0-verification", - "service.role": "verification", - "deployment.environment.name": "staging", - "cloud.platform": "gcp_cloud_run", - "cloud.region": "us-central1", - } - ) - - trace_exporter = OTLPSpanExporter( - endpoint=endpoint, - credentials=credentials, - headers=headers, - timeout=10, - ) - trace_provider = TracerProvider(resource=resource) - trace_provider.add_span_processor(SimpleSpanProcessor(trace_exporter)) - tracer = trace_provider.get_tracer( - "policyengine.observability.verification" - ) - with tracer.start_as_current_span( - "policyengine.observability.verify" - ) as span: - span.set_attribute("policyengine.verification", True) - trace_id = f"{span.get_span_context().trace_id:032x}" - trace_provider.force_flush(timeout_millis=15_000) - trace_provider.shutdown() - - metric_exporter = OTLPMetricExporter( - endpoint=endpoint, - credentials=credentials, - headers=headers, - timeout=10, - ) - metric_reader = PeriodicExportingMetricReader( - metric_exporter, - export_interval_millis=60_000, - export_timeout_millis=10_000, - ) - metric_provider = MeterProvider( - resource=resource, - metric_readers=[metric_reader], - ) - meter = metric_provider.get_meter( - "policyengine.observability.verification" - ) - counter = meter.create_counter(METRIC_NAME) - counter.add(1, {"outcome": "success"}) - metric_provider.shutdown() - - host = urllib.parse.urlparse(endpoint).netloc - channel = grpc.secure_channel(host, credentials) - logs_stub = LogsServiceStub(channel) - log_request = ExportLogsServiceRequest( - resource_logs=[ - ResourceLogs( - resource=ProtoResource( - attributes=[ - KeyValue( - key="service.name", - value=AnyValue( - string_value="policyengine-observability-verification" - ), - ) - ] - ), - scope_logs=[ - ScopeLogs( - log_records=[ - LogRecord( - time_unix_nano=time.time_ns(), - severity_text="INFO", - body=AnyValue( - string_value="collector log rejection verification" - ), - ) - ] - ) - ], - ) - ] - ) - log_status = "accepted" - try: - logs_stub.Export(log_request, timeout=10, metadata=headers) - except grpc.RpcError as error: - log_status = error.code().name - finally: - channel.close() - - return { - "trace_id": trace_id, - "metric_type": METRIC_TYPE, - "log_export_status": log_status, - } - - -def _wait_for_storage( - *, trace_id: str, metric_type: str, timeout_seconds: int -) -> dict[str, object]: - access_token = _access_token() - deadline = time.monotonic() + timeout_seconds - start = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 300)) - end = time.strftime( - "%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() + timeout_seconds + 60) - ) - metric_filter = urllib.parse.quote(f'metric.type = "{metric_type}"') - metric_url = ( - f"https://monitoring.googleapis.com/v3/projects/{PROJECT}/timeSeries" - f"?filter={metric_filter}&interval.startTime={start}" - f"&interval.endTime={end}&view=HEADERS" - ) - trace_url = f"https://cloudtrace.googleapis.com/v1/projects/{PROJECT}/traces/{trace_id}" - trace_found = False - metric_found = False - while time.monotonic() < deadline and not (trace_found and metric_found): - if not trace_found: - try: - trace_payload = _authorized_get(trace_url, access_token) - trace_found = bool(trace_payload.get("spans")) - except urllib.error.HTTPError as error: - if error.code != 404: - raise - if not metric_found: - try: - metric_payload = _authorized_get(metric_url, access_token) - metric_found = bool(metric_payload.get("timeSeries")) - except urllib.error.HTTPError as error: - if error.code != 404: - raise - if not (trace_found and metric_found): - time.sleep(5) - return { - "trace_stored": trace_found, - "metric_stored": metric_found, - } - - -def main() -> int: - if not PROJECT: - raise SystemExit( - "Missing deployment variable: OBSERVABILITY_PROJECT_ID" - ) - parser = argparse.ArgumentParser() - parser.add_argument("--endpoint", required=True) - parser.add_argument("--service-account", required=True) - parser.add_argument("--timeout-seconds", type=int, default=120) - arguments = parser.parse_args() - - # Add a run identifier without placing high-cardinality values on the metric. - run_id = secrets.token_hex(4) - result = _send_signals(arguments.endpoint, arguments.service_account) - result.update( - _wait_for_storage( - trace_id=str(result["trace_id"]), - metric_type=str(result["metric_type"]), - timeout_seconds=arguments.timeout_seconds, - ) - ) - result["run_id"] = run_id - print(json.dumps(result, sort_keys=True)) - - return ( - 0 - if ( - result["trace_stored"] - and result["metric_stored"] - and result["log_export_status"] != "accepted" - ) - else 1 - ) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/deploy/gcp/workload-inventory.template.yaml b/deploy/gcp/workload-inventory.template.yaml deleted file mode 100644 index f74db60..0000000 --- a/deploy/gcp/workload-inventory.template.yaml +++ /dev/null @@ -1,70 +0,0 @@ -schema_version: policyengine.observability.workloads.v1 -destination_project: - project_id: ${OBSERVABILITY_PROJECT_ID} - project_number: "${OBSERVABILITY_PROJECT_NUMBER}" - log_bucket: ${OBSERVABILITY_PROJECT_ID} - log_bucket_location: global - log_retention_days: 30 - -cloud_run: - - service_name: policyengine-api - source_project: ${API_PROJECT_ID} - region: us-central1 - environment: production - service_account: policyengine-api-cr-runtime@${API_PROJECT_ID}.iam.gserviceaccount.com - signals: [logs, traces, metrics] - include_tagged_revisions: true - - service_name: policyengine-api-staging - source_project: ${API_PROJECT_ID} - region: us-central1 - environment: staging - service_account: policyengine-api-cr-staging@${API_PROJECT_ID}.iam.gserviceaccount.com - signals: [logs, traces, metrics] - include_tagged_revisions: true - - service_name: policyengine-simulation-entry - source_project: ${SIMULATION_ENTRY_PROJECT_ID} - region: us-central1 - environment: production - service_account: sim-entry-prod-runtime@${SIMULATION_ENTRY_PROJECT_ID}.iam.gserviceaccount.com - signals: [logs, traces, metrics] - include_tagged_revisions: true - - service_name: policyengine-simulation-entry-staging - source_project: ${SIMULATION_ENTRY_PROJECT_ID} - region: us-central1 - environment: staging - service_account: sim-entry-beta-runtime@${SIMULATION_ENTRY_PROJECT_ID}.iam.gserviceaccount.com - signals: [logs, traces, metrics] - include_tagged_revisions: true - -modal: - workspace_name: policyengine - workspace_id: ${MODAL_WORKSPACE_ID} - environments: - - name: main - deployment_environment: production - - name: staging - deployment_environment: staging - applications: - - match: exact - value: policyengine-simulation-gateway - signals: [logs, traces, metrics] - - match: prefix - value: policyengine-simulation-py - name_constraint: '^policyengine-simulation-py[0-9]+-[0-9]+-[0-9]+$' - signals: [logs, traces, metrics] - -excluded: - modal_exact: - - policyengine-simulation-precompute - - policyengine-simulation-executor-smoke - - policyengine-simulation-gateway-smoke - modal_prefixes: - - policyengine-simulation-gateway-ephemeral- - - policyengine-simulation-py4-12-0-ephemeral- - - policyengine-household-api- - - policyengine-uk-chat - - peukchat- - policy: >- - Every Cloud Run service and Modal application absent from the allowlist is - excluded. Excluded application repositories, deployments, and test suites - are outside this change. diff --git a/docs/engineering/skills/repository-guidance.md b/docs/engineering/skills/repository-guidance.md index 872d237..a3bb86c 100644 --- a/docs/engineering/skills/repository-guidance.md +++ b/docs/engineering/skills/repository-guidance.md @@ -25,19 +25,18 @@ uv run --extra dev towncrier check --compare-with origin/main - `policyengine_observability/config.py` resolves environment-driven runtime configuration. -- `policyengine_observability/context.py` defines request and operation log - payload structures. - `policyengine_observability/runtime.py` preserves the public runtime API, configures the components, and coordinates their shutdown. -- `policyengine_observability/_state.py` owns shared context variables. -- `policyengine_observability/_operations.py` and `_requests.py` manage - operation and request lifecycles, respectively. -- `policyengine_observability/segments.py` manages segment naming, nesting, - and timing. -- `policyengine_observability/logging.py` emits structured logs and records - observability failures without interrupting application operations. -- `policyengine_observability/_metrics.py` and `_tracing.py` record metrics - and manage OpenTelemetry traces, respectively. +- `policyengine_observability/delivery.py` isolates configured log + destinations and gives each remote destination its own bounded queue. +- `policyengine_observability/destinations/` contains the provider-neutral + destination strategy contract and optional built-in destinations. +- `policyengine_observability/schema.py` builds provider-neutral structured + records. Provider fields belong in destination formatters. +- `policyengine_observability/otel.py` records metrics and traces and exports + each enabled signal through its configured OTLP transport. +- `policyengine_observability/google_auth.py` and + `google_credentials.py` contain optional Google authentication strategies. - `policyengine_observability/adapters/` contains framework adapters such as Flask and FastAPI. - `policyengine_observability/integrations/` contains optional integrations @@ -54,6 +53,10 @@ uv run --extra dev towncrier check --compare-with origin/main CLI scripts, and tests. - Keep OpenTelemetry optional and lazily imported. Timing and structured logging must work without an OTel backend. +- Keep canonical records free of provider-specific field names. Apply those + fields in the configured destination formatter or writer. +- Keep OTLP transport independent from exporter authentication and allow + traces and metrics to use different endpoints. - Observability failures must fail open: record an internal observability error when practical, but do not break the application operation being observed. - Preserve structured log schemas. Make additive changes when possible; bump @@ -61,14 +64,14 @@ uv run --extra dev towncrier check --compare-with origin/main - Keep metric attributes bounded and low-cardinality. Do not put raw paths, full URLs, request bodies, or unbounded user-provided values into metric labels. -- Keep segment names stable. Prefer registered segment enums in consuming +- Keep span names stable. Prefer registered span enums in consuming applications, while preserving safe string fallback behavior. ## Testing Add focused tests for context behavior and failure paths whenever changing the runtime or its components. The corresponding `tests/test_runtime_*.py` -modules cover operations, requests, segments, log emission, and tracing. +modules cover operations, requests, spans, log emission, and tracing. Adapter changes should include framework-level tests that exercise request setup, response headers, error paths, and teardown behavior. diff --git a/docs/operations/api-v1-observability.md b/docs/operations/api-v1-observability.md deleted file mode 100644 index 58960ac..0000000 --- a/docs/operations/api-v1-observability.md +++ /dev/null @@ -1,153 +0,0 @@ -# API v1 observability operating policy - -## Scope - -The machine-readable workload inventory is -[`deploy/gcp/workload-inventory.template.yaml`](../../deploy/gcp/workload-inventory.template.yaml). -Only the listed `policyengine-api`, simulation entry, simulation gateway, and -versioned simulation executor workloads participate. - -Cloud Run candidate, canary, and tagged revisions use the identity of their -containing service and are included. Modal smoke, precompute, and ephemeral -applications are excluded. `policyengine-household-api` and -`policyengine-uk-chat` remain unchanged and receive no migration or -service-specific validation in this work. - -The inventory is the configuration source for log sink filters, collector -invocation permissions, and the Modal Workload Identity Federation condition. -Telemetry attributes such as `service.namespace` do not grant access. - -## Storage decision - -Production and nonproduction application logs use the existing -`policyengine-observability` bucket in the `policyengine-observability` Google -Cloud project. The bucket is global, analytics enabled, and retains records for -30 days. `deployment.environment.name` distinguishes production and staging. - -A single bucket is appropriate for the initial rollout because the same small -operator group requires both environments, the current bucket already exists, -and a shared analytics surface simplifies request investigations. Access is -controlled at the bucket and project level rather than by environment. This -decision must be revisited before an environment requires different readers, -retention, residency, or deletion policy. - -Cloud Run writes structured JSON to standard output. Exact-service sinks in -the source projects route selected records into this bucket. Modal writes the -same records to standard output and uses the package's bounded asynchronous -Cloud Logging destination under the `policyengine-api-v1-modal` log ID. A -central exclusion prevents a directly ingested record from also being retained -in `_Default`. - -Traces and metrics use Cloud Trace and Cloud Monitoring in the same project. -They are correlated with logs by resource identity, trace ID, request ID, and -job ID; they are not stored in the log bucket. - -## Initial trace sampling - -The initial rollout uses these head-sampling settings: - -| Environment | Services | Parent-based ratio | -| --- | --- | ---: | -| Production | API v1, simulation entry, gateway, executors | 1.0 | -| Staging | API v1, simulation entry, gateway, executors | 1.0 | -| Local development | All | No remote exporter | - -Sampling every initial production trace provides a complete baseline for -volume, cost, errors, and slow operations. After at least one representative -week, operators may lower the normal-request ratio only after recording a cost -and coverage review. A lower head-sampling ratio cannot retroactively retain a -request after its outcome or duration becomes known. Retaining all errors or -slow requests with a lower normal ratio therefore requires a reviewed, -bounded collector tail-sampling policy. - -The checked-in collector configuration implements error and 30-second latency -policies and initially retains 100% of all remaining traces. Any later -reduction applies to the collector's general tail policy while SDK head -sampling remains at 100%, allowing the collector to evaluate completed traces. - -Parent sampling decisions are preserved. Society-wide simulation dispatches -must not override a sampled parent with an unsampled child. - -## Asynchronous trace relationships - -Captured dispatch context contains its UTC capture time. A worker uses the -dispatch span as its parent only when all of these conditions hold: - -- The work is a direct continuation of one dispatch. -- The worker starts no more than five minutes after capture. -- The invocation is not an independent retry. -- The invocation does not aggregate multiple dispatches. - -Otherwise the worker starts a new trace and links the dispatch span. Request -and job identifiers remain the same in either representation. Malformed or -expired remote context is ignored without rejecting the job. - -## Data policy - -### Required log fields - -- `schema_version` -- `timestamp` -- `severity` -- `message` or `event.name` -- `service.name`, `service.namespace`, `service.version`, `service.role` -- `deployment.environment.name` and `cloud.platform` -- Request, operation, trace, span, duration, outcome, and bounded error fields - when applicable - -Application attributes are stored below `attributes`. The initial allowlist is -limited to bounded operational values such as country, model version, backend, -requested version, resolved channel, authentication outcome, job type, and -simulation year. Attribute strings are truncated at 1,024 characters and one -record contains at most 32 application attributes. - -### Trace attributes - -Traces may contain the standard service resource fields, HTTP route templates, -HTTP methods, status codes, operation names, bounded deployment identifiers, -request IDs, job IDs, and explicitly approved operational attributes. Raw URLs, -query values, request bodies, response bodies, and arbitrary baggage are not -recorded. - -### Metric labels - -Metrics use only these bounded labels: - -- `service.name` -- `service.role` -- `deployment.environment.name` -- `cloud.platform` -- `http.route` -- `http.request.method` -- `http.response.status_code_class` -- `operation.name` -- `operation.kind` -- `outcome` - -Request IDs, trace IDs, job IDs, simulation IDs, raw paths, error messages, -user-provided values, unrestricted geography values, and unrestricted version -values are prohibited metric labels. - -### Prohibited telemetry data - -Logs, traces, metrics, and dispatch context must not contain: - -- Authorization headers, cookies, credentials, tokens, or secret values -- Request or response bodies -- Household situations, entity records, or person-level values -- Reform definitions or parameter payloads -- Raw IP addresses -- Prompts, model inputs, or model responses -- Exception local variables -- Function arguments or return values captured automatically - -Exception messages and stacks are truncated and passed through configured -secret-value redaction before remote delivery. - -## Operational limits - -Remote application delivery is best effort. Every application queue, exporter, -retry, network request, flush, and shutdown action has a finite bound. Queue -overflow drops the new record and increments a local counter. Internal -diagnostics are rate limited and written directly to standard error so they do -not recurse through a failing exporter. diff --git a/policyengine_observability/__init__.py b/policyengine_observability/__init__.py index c82570e..224b865 100644 --- a/policyengine_observability/__init__.py +++ b/policyengine_observability/__init__.py @@ -3,13 +3,23 @@ from .adapters import instrument_fastapi, instrument_flask from .config import ( DeploymentIdentity, - GoogleCloudLoggingConfig, LoggingConfig, ObservabilityConfig, OTelConfig, + OTLPAuthentication, + OTLPExporterConfig, ServiceIdentity, TelemetryLimits, ) +from .destinations import ( + CustomLogDestination, + GoogleCloudLogDestination, + GoogleCloudLogFormatter, + LogDestinationStrategy, + RecordWriter, + StdoutLogDestination, +) +from .google_auth import GoogleIdTokenAuth from .integrations import instrument_httpx from .runtime import ( REQUEST_ID_HEADER, @@ -27,15 +37,23 @@ "SCHEMA_VERSION", "TRACEPARENT_HEADER", "TRACESTATE_HEADER", + "CustomLogDestination", "DeploymentIdentity", - "GoogleCloudLoggingConfig", + "GoogleCloudLogDestination", + "GoogleCloudLogFormatter", + "GoogleIdTokenAuth", + "LogDestinationStrategy", "LoggingConfig", + "OTLPAuthentication", + "OTLPExporterConfig", "OTelConfig", "ObservabilityConfig", "ObservabilityLogHandler", "ObservabilityRuntime", + "RecordWriter", "ServiceIdentity", "TelemetryLimits", + "StdoutLogDestination", "configure", "instrument_fastapi", "instrument_flask", diff --git a/policyengine_observability/config.py b/policyengine_observability/config.py index f64aa1e..c51e150 100644 --- a/policyengine_observability/config.py +++ b/policyengine_observability/config.py @@ -2,32 +2,17 @@ import os from dataclasses import dataclass, field -from typing import Literal, cast +from typing import Any, Literal, Protocol, cast + +from .destinations import LogDestinationStrategy, StdoutLogDestination Platform = Literal["google_cloud_run", "modal", "local", "other"] OTLPProtocol = Literal["grpc", "http/protobuf"] ProviderMode = Literal["owned", "external"] -DEFAULT_APPLICATION_ATTRIBUTE_KEYS = frozenset( - { - "auth_result", - "backend", - "country_id", - "job_type", - "model_version", - "requested_version", - "resolved_channel", - "simulation_year", - } -) +DEFAULT_APPLICATION_ATTRIBUTE_KEYS = frozenset(set()) -DEFAULT_DISPATCH_ATTRIBUTE_KEYS = frozenset( - { - "job_id", - "run_id", - "simulation_id", - } -) +DEFAULT_DISPATCH_ATTRIBUTE_KEYS = frozenset(set()) DEFAULT_METRIC_ATTRIBUTE_KEYS = frozenset( { @@ -61,40 +46,47 @@ class DeploymentIdentity: instance_id: str | None = None -@dataclass(frozen=True, slots=True) -class GoogleCloudLoggingConfig: - project_id: str - log_name: str = "policyengine-api-v1-modal" - queue_capacity: int = 1_000 - batch_size: int = 1 - write_timeout_seconds: float = 5.0 - - @dataclass(frozen=True, slots=True) class LoggingConfig: - stdout_enabled: bool = True - remote: GoogleCloudLoggingConfig | None = None + destinations: tuple[LogDestinationStrategy, ...] = field( + default_factory=lambda: (StdoutLogDestination(),) + ) capture_standard_library: bool = False replace_existing_handlers: bool = False minimum_severity: int = 20 shutdown_timeout_seconds: float = 2.0 +class OTLPAuthentication(Protocol): + """Adds authentication-specific arguments to an OTLP exporter.""" + + def exporter_kwargs( + self, + *, + protocol: OTLPProtocol, + headers: dict[str, str], + ) -> dict[str, Any]: ... + + @dataclass(frozen=True, slots=True) -class OTelConfig: - enabled: bool = True - traces_enabled: bool = True - metrics_enabled: bool = True - endpoint: str | None = None +class OTLPExporterConfig: + endpoint: str protocol: OTLPProtocol = "grpc" headers: tuple[tuple[str, str], ...] = () - google_audience: str | None = None + auth: OTLPAuthentication | None = None + timeout_seconds: float = 5.0 + + +@dataclass(frozen=True, slots=True) +class OTelConfig: + enabled: bool = True + traces: OTLPExporterConfig | None = None + metrics: OTLPExporterConfig | None = None provider_mode: ProviderMode = "owned" sampling_ratio: float = 1.0 span_queue_capacity: int = 2_048 span_batch_size: int = 512 span_schedule_delay_seconds: float = 5.0 - export_timeout_seconds: float = 5.0 metric_export_interval_seconds: float = 60.0 shutdown_timeout_seconds: float = 3.0 @@ -112,7 +104,6 @@ class TelemetryLimits: class ObservabilityConfig: service: ServiceIdentity deployment: DeploymentIdentity - google_cloud_project_id: str | None = None logging: LoggingConfig = field(default_factory=LoggingConfig) otel: OTelConfig = field(default_factory=OTelConfig) limits: TelemetryLimits = field(default_factory=TelemetryLimits) @@ -130,7 +121,6 @@ def from_env( service: ServiceIdentity, deployment: DeploymentIdentity, logging: LoggingConfig | None = None, - google_cloud_project_id: str | None = None, limits: TelemetryLimits | None = None, application_attribute_keys: frozenset[str] | None = None, dispatch_attribute_keys: frozenset[str] | None = None, @@ -143,28 +133,37 @@ def from_env( inferred from ambient platform or Google Cloud variables. """ - endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") - protocol = os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").strip() - if protocol not in {"grpc", "http/protobuf"}: - protocol = "grpc" + common_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") + common_protocol = _protocol( + os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc") + ) + common_headers = os.getenv("OTEL_EXPORTER_OTLP_HEADERS", "") + common_audience = os.getenv("POLICYENGINE_OTEL_GOOGLE_AUDIENCE") + traces = _exporter_from_env( + signal="traces", + enabled=os.getenv("OTEL_TRACES_EXPORTER", "otlp") != "none", + common_endpoint=common_endpoint, + common_protocol=common_protocol, + common_headers=common_headers, + common_audience=common_audience, + ) + metrics = _exporter_from_env( + signal="metrics", + enabled=os.getenv("OTEL_METRICS_EXPORTER", "otlp") != "none", + common_endpoint=common_endpoint, + common_protocol=common_protocol, + common_headers=common_headers, + common_audience=common_audience, + ) return cls( service=service, deployment=deployment, - google_cloud_project_id=google_cloud_project_id, logging=logging or LoggingConfig(), otel=OTelConfig( enabled=not _env_bool("OTEL_SDK_DISABLED", False), - traces_enabled=os.getenv("OTEL_TRACES_EXPORTER", "otlp") - != "none", - metrics_enabled=os.getenv("OTEL_METRICS_EXPORTER", "otlp") - != "none", - endpoint=endpoint, - protocol=cast(OTLPProtocol, protocol), - headers=_parse_headers( - os.getenv("OTEL_EXPORTER_OTLP_HEADERS", "") - ), - google_audience=os.getenv("POLICYENGINE_OTEL_GOOGLE_AUDIENCE"), + traces=traces, + metrics=metrics, provider_mode=_provider_mode( os.getenv("POLICYENGINE_OTEL_PROVIDER_MODE", "owned") ), @@ -195,15 +194,6 @@ def from_env( ) / 1_000 ), - export_timeout_seconds=( - _bounded_float( - os.getenv("OTEL_BSP_EXPORT_TIMEOUT"), - default=5_000.0, - minimum=1.0, - maximum=60_000.0, - ) - / 1_000 - ), metric_export_interval_seconds=( _bounded_float( os.getenv("OTEL_METRIC_EXPORT_INTERVAL"), @@ -247,23 +237,28 @@ def diagnostics(self) -> tuple[str, ...]: if not str(value).strip(): messages.append(f"Missing explicit runtime identity: {key}") - remote = self.logging.remote - if remote is not None and ( - not remote.project_id.strip() or not remote.log_name.strip() + for destination in self.logging.destinations: + validate = getattr(destination, "diagnostics", None) + if not callable(validate): + continue + try: + messages.extend( + str(item) for item in cast(tuple[str, ...], validate()) + ) + except Exception as exc: + messages.append( + "Log destination validation failed for " + f"{getattr(destination, 'name', '')}: {exc}" + ) + if ( + self.otel.enabled + and self.otel.provider_mode == "owned" + and self.otel.traces is None + and self.otel.metrics is None ): messages.append( - "Remote logging is disabled because its project or log name " - "is empty." - ) - if remote is not None and self.deployment.platform != "modal": - messages.append( - "Direct Cloud Logging is disabled outside Modal; use platform " - "standard-output ingestion." - ) - if self.otel.enabled and not self.otel.endpoint: - messages.append( - "Remote OTel export is disabled because no OTLP endpoint is " - "configured." + "Remote OTel export is disabled because no trace or metric " + "OTLP endpoint is configured." ) return tuple(messages) @@ -293,6 +288,63 @@ def _provider_mode(value: str) -> ProviderMode: return "external" if value.strip().lower() == "external" else "owned" +def _protocol(value: str) -> OTLPProtocol: + normalized = value.strip() + if normalized not in {"grpc", "http/protobuf"}: + return "grpc" + return cast(OTLPProtocol, normalized) + + +def _exporter_from_env( + *, + signal: Literal["traces", "metrics"], + enabled: bool, + common_endpoint: str | None, + common_protocol: OTLPProtocol, + common_headers: str, + common_audience: str | None, +) -> OTLPExporterConfig | None: + if not enabled: + return None + prefix = f"OTEL_EXPORTER_OTLP_{signal.upper()}" + endpoint = os.getenv(f"{prefix}_ENDPOINT") or common_endpoint + if not endpoint: + return None + protocol_value = os.getenv(f"{prefix}_PROTOCOL") + protocol = ( + _protocol(protocol_value) + if protocol_value is not None + else common_protocol + ) + headers_value = os.getenv(f"{prefix}_HEADERS") + headers = _parse_headers( + headers_value if headers_value is not None else common_headers + ) + audience = ( + os.getenv(f"POLICYENGINE_OTEL_{signal.upper()}_GOOGLE_AUDIENCE") + or common_audience + ) + auth: OTLPAuthentication | None = None + if audience: + from .google_auth import GoogleIdTokenAuth + + auth = GoogleIdTokenAuth(audience) + timeout = _bounded_float( + os.getenv(f"{prefix}_TIMEOUT") + or os.getenv("OTEL_EXPORTER_OTLP_TIMEOUT"), + default=5_000.0, + minimum=100.0, + maximum=60_000.0, + ) + return OTLPExporterConfig( + endpoint=endpoint, + protocol=protocol, + headers=headers, + auth=auth, + timeout_seconds=timeout / 1_000, + ) + + def _parse_headers(value: str) -> tuple[tuple[str, str], ...]: headers: list[tuple[str, str]] = [] for item in value.split(","): diff --git a/policyengine_observability/delivery.py b/policyengine_observability/delivery.py index 0805ddd..3da0e14 100644 --- a/policyengine_observability/delivery.py +++ b/policyengine_observability/delivery.py @@ -1,89 +1,175 @@ from __future__ import annotations -import functools -import json import queue import sys import threading -from collections.abc import Callable -from typing import Any, Protocol, TextIO +import time +from dataclasses import dataclass +from typing import Any, TextIO -from .config import GoogleCloudLoggingConfig, ObservabilityConfig +from .config import ObservabilityConfig +from .destinations import ( + DestinationBuildContext, + LogDestinationStrategy, + RecordWriter, + StdoutLogDestination, +) from .diagnostics import Diagnostics +DESTINATION_FAILURE_LIMIT = 3 -class RecordWriter(Protocol): - def write(self, record: dict[str, Any]) -> None: ... - def write_many(self, records: list[dict[str, Any]]) -> None: ... - - def close(self) -> None: ... +@dataclass(slots=True) +class _InlineDestination: + strategy: LogDestinationStrategy + writer: RecordWriter + failures: int = 0 class DeliveryManager: + """Fan out provider-neutral records to independently isolated writers.""" + def __init__( self, config: ObservabilityConfig, diagnostics: Diagnostics, *, stdout: TextIO | None = None, - writer_factory: Callable[[GoogleCloudLoggingConfig], RecordWriter] - | None = None, ) -> None: self.config = config self.diagnostics = diagnostics self._stdout = stdout or sys.stdout - self._writer_factory = writer_factory or _GoogleCloudWriter - self._remote: _QueuedWriter | None = None - remote = config.logging.remote - if ( - remote is not None - and config.deployment.platform == "modal" - and config.identity_complete - and remote.project_id.strip() - and remote.log_name.strip() - ): - self._remote = _QueuedWriter( - remote, - diagnostics, - writer_factory=self._writer_factory, + self._inline: list[_InlineDestination] = [] + self._queued: list[_QueuedWriter] = [] + self._configure() + + def _configure(self) -> None: + strategies = self.config.logging.destinations + if not strategies: + strategies = (StdoutLogDestination(),) + self.diagnostics.report( + "logging.destination_config", + "No log destination was configured; using standard output.", + ) + context = DestinationBuildContext(stdout=lambda: self._stdout) + for strategy in strategies: + try: + self._add_strategy(strategy, context) + except Exception as exc: + self.diagnostics.report( + "logging.destination_config", + exc, + destination=getattr(strategy, "name", None), + ) + if not self._inline and not self._queued: + self._add_strategy(StdoutLogDestination(), context) + + def _add_strategy( + self, + strategy: LogDestinationStrategy, + context: DestinationBuildContext, + ) -> None: + if strategy.delivery == "inline": + self._inline.append( + _InlineDestination(strategy, strategy.build_writer(context)) + ) + return + if strategy.delivery != "queued": + raise ValueError( + f"Unsupported delivery mode for {strategy.name!r}: " + f"{strategy.delivery!r}" + ) + self._queued.append( + _QueuedWriter( + strategy, + context, + self.diagnostics, ) + ) def emit(self, record: dict[str, Any]) -> None: - if self.config.logging.stdout_enabled: + disabled: list[_InlineDestination] = [] + for destination in tuple(self._inline): try: - print( - json.dumps( - record, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=True, - ), - file=self._stdout, - flush=True, - ) + destination.writer.write(record.copy()) + destination.failures = 0 except Exception as exc: - self.diagnostics.report("stdout.write", exc) - if self._remote is not None: - self._remote.enqueue(record.copy()) + destination.failures += 1 + self.diagnostics.increment("logs.export_failure") + self.diagnostics.report( + "logging.export", + exc, + destination=destination.strategy.name, + consecutive_failures=destination.failures, + ) + if destination.failures >= DESTINATION_FAILURE_LIMIT: + disabled.append(destination) + for destination in disabled: + self._disable_inline(destination) + for destination in tuple(self._queued): + destination.enqueue(record.copy()) - def close(self, timeout_seconds: float | None = None) -> None: - if self._remote is None: + def _disable_inline(self, destination: _InlineDestination) -> None: + try: + self._inline.remove(destination) + except ValueError: return + self._close_writer(destination.writer, destination.strategy.name) + self.diagnostics.report( + "logging.destination_disabled", + RuntimeError( + "Log destination disabled after repeated write failures." + ), + destination=destination.strategy.name, + ) + if self._inline or self._queued: + return + try: + context = DestinationBuildContext(stdout=lambda: self._stdout) + fallback = StdoutLogDestination() + self._inline.append( + _InlineDestination(fallback, fallback.build_writer(context)) + ) + except Exception as exc: + self.diagnostics.report("logging.stdout_fallback", exc) + + def close(self, timeout_seconds: float | None = None) -> None: timeout = ( self.config.logging.shutdown_timeout_seconds if timeout_seconds is None else timeout_seconds ) - self._remote.close(timeout) + try: + timeout = min(max(float(timeout), 0.0), 30.0) + except (TypeError, ValueError): + timeout = 2.0 + deadline = time.monotonic() + timeout + for destination in tuple(self._queued): + destination.close(max(0.0, deadline - time.monotonic())) + for destination in tuple(self._inline): + self._close_writer( + destination.writer, + destination.strategy.name, + ) + + def _close_writer(self, writer: RecordWriter, name: str) -> None: + close = getattr(writer, "close", None) + if not callable(close): + return + try: + close() + except Exception as exc: + self.diagnostics.report( + "logging.writer_close", exc, destination=name + ) @property def remote_enabled(self) -> bool: - return self._remote is not None + return bool(self._queued) @property def queue_depth(self) -> int: - return self._remote.queue_depth if self._remote else 0 + return sum(destination.queue_depth for destination in self._queued) class _QueuedWriter: @@ -91,22 +177,25 @@ class _QueuedWriter: def __init__( self, - config: GoogleCloudLoggingConfig, + strategy: LogDestinationStrategy, + context: DestinationBuildContext, diagnostics: Diagnostics, - *, - writer_factory: Callable[[GoogleCloudLoggingConfig], RecordWriter], ) -> None: - self.config = config + self.strategy = strategy + self.context = context self.diagnostics = diagnostics - self._writer_factory = writer_factory + capacity = max( + 1, + min(int(getattr(strategy, "queue_capacity", 1_000)), 100_000), + ) self._queue: queue.Queue[dict[str, Any] | object] = queue.Queue( - maxsize=max(1, min(config.queue_capacity, 100_000)) + maxsize=capacity ) self._closed = threading.Event() self._writer: RecordWriter | None = None self._thread = threading.Thread( target=self._run, - name="policyengine-observability-log-writer", + name=f"policyengine-observability-{strategy.name}", daemon=True, ) self._thread.start() @@ -122,6 +211,7 @@ def enqueue(self, record: dict[str, Any]) -> None: self.diagnostics.report( "logging.queue_full", "Remote log queue is full; newest record dropped.", + destination=self.strategy.name, capacity=self._queue.maxsize, ) @@ -143,6 +233,7 @@ def close(self, timeout_seconds: float) -> None: self.diagnostics.report( "logging.shutdown_timeout", "Remote log worker did not stop before its deadline.", + destination=self.strategy.name, ) def _run(self) -> None: @@ -159,9 +250,14 @@ def _run(self) -> None: batch: list[dict[str, Any]] = [] if isinstance(item, dict): batch.append(item) - while not stop_after_batch and len(batch) < max( - 1, self.config.batch_size - ): + batch_size = max( + 1, + min( + int(getattr(self.strategy, "batch_size", 1)), + 10_000, + ), + ) + while not stop_after_batch and len(batch) < batch_size: try: next_item = self._queue.get_nowait() except queue.Empty: @@ -175,7 +271,7 @@ def _run(self) -> None: if not batch: return if self._writer is None: - self._writer = self._writer_factory(self.config) + self._writer = self.strategy.build_writer(self.context) write_many = getattr(self._writer, "write_many", None) if callable(write_many): write_many(batch) @@ -184,89 +280,25 @@ def _run(self) -> None: self._writer.write(record) except Exception as exc: self.diagnostics.increment("logs.export_failure") - self.diagnostics.report("logging.export", exc) + self.diagnostics.report( + "logging.export", + exc, + destination=self.strategy.name, + ) finally: - for _consumed_item in consumed: + for _item in consumed: self._queue.task_done() if stop_after_batch: return finally: if self._writer is not None: - try: - self._writer.close() - except Exception as exc: - self.diagnostics.report("logging.writer_close", exc) - - -class _GoogleCloudWriter: - """Lazy worker-thread-only Cloud Logging client.""" - - def __init__(self, config: GoogleCloudLoggingConfig) -> None: - from google.cloud import logging_v2 - - from .google_credentials import load_google_credentials - - credentials = load_google_credentials(prefer_workload_identity=True) - self._client = logging_v2.Client( - project=config.project_id, - credentials=credentials, - ) - self._logger = self._client.logger(config.log_name) - self._timeout = max(0.1, min(config.write_timeout_seconds, 60.0)) - self._bound_write_timeout() - logging_v2._instrumentation_emitted = True - - def write(self, record: dict[str, Any]) -> None: - self.write_many([record]) - - def write_many(self, records: list[dict[str, Any]]) -> None: - batch = self._logger.batch() - for record in records: - kwargs = self._entry_kwargs(record) - batch.log_struct(record, **kwargs) - batch.commit() - - def _entry_kwargs(self, record: dict[str, Any]) -> dict[str, Any]: - kwargs: dict[str, Any] = { - "severity": record.get("severity", "DEFAULT"), - } - trace = record.get("logging.googleapis.com/trace") - span_id = record.get("logging.googleapis.com/spanId") - if trace: - kwargs["trace"] = trace - if span_id: - kwargs["span_id"] = span_id - kwargs["trace_sampled"] = bool( - record.get("logging.googleapis.com/trace_sampled", False) - ) - return kwargs - - def close(self) -> None: - close = getattr(self._client, "close", None) - if callable(close): - close() - - def _bound_write_timeout(self) -> None: - api = getattr(self._client, "logging_api", None) - gapic = getattr(api, "_gapic_api", None) - if gapic is None: - return - from google.api_core import exceptions as api_exceptions - from google.api_core.retry import Retry, if_exception_type - - retry = Retry( - initial=0.1, - maximum=1.0, - multiplier=1.3, - timeout=self._timeout, - predicate=if_exception_type( - api_exceptions.DeadlineExceeded, - api_exceptions.InternalServerError, - api_exceptions.ServiceUnavailable, - ), - ) - gapic.write_log_entries = functools.partial( - gapic.write_log_entries, - retry=retry, - timeout=self._timeout, - ) + close = getattr(self._writer, "close", None) + if callable(close): + try: + close() + except Exception as exc: + self.diagnostics.report( + "logging.writer_close", + exc, + destination=self.strategy.name, + ) diff --git a/policyengine_observability/destinations/__init__.py b/policyengine_observability/destinations/__init__.py new file mode 100644 index 0000000..f827215 --- /dev/null +++ b/policyengine_observability/destinations/__init__.py @@ -0,0 +1,25 @@ +"""Logging destination strategies supplied to :class:`LoggingConfig`.""" + +from .base import ( + CustomLogDestination, + DestinationBuildContext, + LogDestinationStrategy, + RecordFormatter, + RecordWriter, +) +from .google_cloud import ( + GoogleCloudLogDestination, + GoogleCloudLogFormatter, +) +from .stdout import StdoutLogDestination + +__all__ = [ + "CustomLogDestination", + "DestinationBuildContext", + "GoogleCloudLogDestination", + "GoogleCloudLogFormatter", + "LogDestinationStrategy", + "RecordFormatter", + "RecordWriter", + "StdoutLogDestination", +] diff --git a/policyengine_observability/destinations/base.py b/policyengine_observability/destinations/base.py new file mode 100644 index 0000000..3441335 --- /dev/null +++ b/policyengine_observability/destinations/base.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any, Literal, Protocol, TextIO, runtime_checkable + +DeliveryMode = Literal["inline", "queued"] +RecordFormatter = Callable[[dict[str, Any]], dict[str, Any]] + + +class RecordWriter(Protocol): + """Writes provider-neutral structured records to one destination.""" + + def write(self, record: dict[str, Any]) -> None: ... + + +@dataclass(frozen=True, slots=True) +class DestinationBuildContext: + """Process-local resources available while constructing a writer.""" + + stdout: Callable[[], TextIO] + + +@runtime_checkable +class LogDestinationStrategy(Protocol): + """Configuration and factory contract for a logging destination.""" + + @property + def name(self) -> str: ... + + @property + def delivery(self) -> DeliveryMode: ... + + @property + def queue_capacity(self) -> int: ... + + @property + def batch_size(self) -> int: ... + + def build_writer( + self, context: DestinationBuildContext + ) -> RecordWriter: ... + + +class _FormattingWriter: + def __init__( + self, + writer: RecordWriter, + formatter: RecordFormatter | None, + ) -> None: + self._writer = writer + self._formatter = formatter + + def write(self, record: dict[str, Any]) -> None: + self._writer.write(self._format(record)) + + def write_many(self, records: list[dict[str, Any]]) -> None: + formatted = [self._format(record) for record in records] + write_many = getattr(self._writer, "write_many", None) + if callable(write_many): + write_many(formatted) + return + for record in formatted: + self._writer.write(record) + + def close(self) -> None: + close = getattr(self._writer, "close", None) + if callable(close): + close() + + def _format(self, record: dict[str, Any]) -> dict[str, Any]: + value = record.copy() + return self._formatter(value) if self._formatter else value + + +@dataclass(frozen=True, slots=True) +class CustomLogDestination: + """Adapts an application or third-party writer into the runtime. + + Queued delivery is the safe default. Use inline delivery only for writers + that perform bounded local work and never make network requests. + """ + + name: str + writer_factory: Callable[[], RecordWriter] + delivery: DeliveryMode = "queued" + queue_capacity: int = 1_000 + batch_size: int = 1 + formatter: RecordFormatter | None = None + + def build_writer(self, _context: DestinationBuildContext) -> RecordWriter: + return _FormattingWriter(self.writer_factory(), self.formatter) diff --git a/policyengine_observability/destinations/google_cloud.py b/policyengine_observability/destinations/google_cloud.py new file mode 100644 index 0000000..bd97313 --- /dev/null +++ b/policyengine_observability/destinations/google_cloud.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import functools +from dataclasses import dataclass +from typing import Any, Literal + +from .base import DestinationBuildContext, RecordWriter + +GOOGLE_TRACE_KEY = "logging.googleapis.com/trace" +GOOGLE_SPAN_ID_KEY = "logging.googleapis.com/spanId" +GOOGLE_TRACE_SAMPLED_KEY = "logging.googleapis.com/trace_sampled" + + +@dataclass(frozen=True, slots=True) +class GoogleCloudLogFormatter: + """Adds Cloud Logging trace-correlation fields to a record copy.""" + + project_id: str + + def __call__(self, record: dict[str, Any]) -> dict[str, Any]: + trace_id = record.get("trace_id") + span_id = record.get("span_id") + if trace_id and self.project_id.strip(): + record[GOOGLE_TRACE_KEY] = ( + f"projects/{self.project_id}/traces/{trace_id}" + ) + record[GOOGLE_TRACE_SAMPLED_KEY] = bool( + record.get("trace_sampled", False) + ) + if span_id: + record[GOOGLE_SPAN_ID_KEY] = str(span_id) + return record + + +@dataclass(frozen=True, slots=True) +class GoogleCloudLogDestination: + """Writes records directly to the Google Cloud Logging API.""" + + project_id: str + log_name: str + queue_capacity: int = 1_000 + batch_size: int = 100 + write_timeout_seconds: float = 5.0 + name: str = "google_cloud" + delivery: Literal["queued"] = "queued" + + def build_writer(self, _context: DestinationBuildContext) -> RecordWriter: + if not self.project_id.strip() or not self.log_name.strip(): + raise ValueError( + "Google Cloud project_id and log_name must be non-empty." + ) + return _GoogleCloudWriter(self) + + def diagnostics(self) -> tuple[str, ...]: + if self.project_id.strip() and self.log_name.strip(): + return () + return ( + "Google Cloud logging is disabled because project_id or " + "log_name is empty.", + ) + + +class _GoogleCloudWriter: + """Lazy-importing Cloud Logging writer used only by a queue worker.""" + + def __init__(self, config: GoogleCloudLogDestination) -> None: + from google.cloud import logging_v2 + + from ..google_credentials import load_google_credentials + + credentials = load_google_credentials(prefer_workload_identity=True) + self._client = logging_v2.Client( + project=config.project_id, + credentials=credentials, + ) + self._logger = self._client.logger(config.log_name) + self._project_id = config.project_id + self._timeout = max( + 0.1, min(float(config.write_timeout_seconds), 60.0) + ) + self._bound_write_timeout() + logging_v2._instrumentation_emitted = True + + def write(self, record: dict[str, Any]) -> None: + self.write_many([record]) + + def write_many(self, records: list[dict[str, Any]]) -> None: + batch = self._logger.batch() + for record in records: + batch.log_struct(record, **self._entry_kwargs(record)) + batch.commit() + + def _entry_kwargs(self, record: dict[str, Any]) -> dict[str, Any]: + kwargs: dict[str, Any] = { + "severity": record.get("severity", "DEFAULT"), + } + trace_id = record.get("trace_id") + span_id = record.get("span_id") + if trace_id: + kwargs["trace"] = f"projects/{self._project_id}/traces/{trace_id}" + if span_id: + kwargs["span_id"] = str(span_id) + kwargs["trace_sampled"] = bool(record.get("trace_sampled", False)) + return kwargs + + def close(self) -> None: + close = getattr(self._client, "close", None) + if callable(close): + close() + + def _bound_write_timeout(self) -> None: + api = getattr(self._client, "logging_api", None) + gapic = getattr(api, "_gapic_api", None) + if gapic is None: + return + from google.api_core import exceptions as api_exceptions + from google.api_core.retry import Retry, if_exception_type + + retry = Retry( + initial=0.1, + maximum=1.0, + multiplier=1.3, + timeout=self._timeout, + predicate=if_exception_type( + api_exceptions.DeadlineExceeded, + api_exceptions.InternalServerError, + api_exceptions.ServiceUnavailable, + ), + ) + gapic.write_log_entries = functools.partial( + gapic.write_log_entries, + retry=retry, + timeout=self._timeout, + ) diff --git a/policyengine_observability/destinations/stdout.py b/policyengine_observability/destinations/stdout.py new file mode 100644 index 0000000..a7347ec --- /dev/null +++ b/policyengine_observability/destinations/stdout.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any, Literal + +from .base import ( + DestinationBuildContext, + RecordFormatter, + RecordWriter, +) + + +class _StdoutWriter: + def __init__( + self, + context: DestinationBuildContext, + formatter: RecordFormatter | None, + ) -> None: + self._context = context + self._formatter = formatter + + def write(self, record: dict[str, Any]) -> None: + value = record.copy() + if self._formatter is not None: + value = self._formatter(value) + print( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ), + file=self._context.stdout(), + flush=True, + ) + + +@dataclass(frozen=True, slots=True) +class StdoutLogDestination: + """Writes one JSON object per line to the process standard output.""" + + formatter: RecordFormatter | None = None + name: str = "stdout" + delivery: Literal["inline"] = "inline" + queue_capacity: int = 1 + batch_size: int = 1 + + def build_writer(self, context: DestinationBuildContext) -> RecordWriter: + return _StdoutWriter(context, self.formatter) diff --git a/policyengine_observability/google_auth.py b/policyengine_observability/google_auth.py new file mode 100644 index 0000000..de3148b --- /dev/null +++ b/policyengine_observability/google_auth.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import os +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from .config import OTLPProtocol + + +@dataclass(frozen=True, slots=True) +class GoogleIdTokenAuth: + """Authenticates an OTLP exporter to an ID-token protected endpoint.""" + + audience: str + + def exporter_kwargs( + self, + *, + protocol: OTLPProtocol, + headers: dict[str, str], + ) -> dict[str, Any]: + if protocol == "grpc": + return {"credentials": _google_grpc_credentials(self.audience)} + return {"session": _google_http_session(self.audience, headers)} + + +def _google_id_token_credentials(audience: str) -> Any: + from google.auth.transport.requests import Request + + modal_token = os.getenv("MODAL_IDENTITY_TOKEN") or os.getenv( + "OBSERVABILITY_GOOGLE_OIDC_TOKEN" + ) + provider = os.getenv("OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER") + service_account = os.getenv("OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL") + if modal_token and provider and service_account: + from google.auth import identity_pool, impersonated_credentials + + source = identity_pool.Credentials.from_info( + { + "type": "external_account", + "audience": _workload_identity_audience(provider), + "subject_token_type": ("urn:ietf:params:oauth:token-type:jwt"), + "token_url": "https://sts.googleapis.com/v1/token", + "credential_source": { + "file": _write_subject_token(modal_token), + "format": {"type": "text"}, + }, + }, + scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + target = impersonated_credentials.Credentials( + source_credentials=source, + target_principal=service_account, + target_scopes=["https://www.googleapis.com/auth/cloud-platform"], + ) + return impersonated_credentials.IDTokenCredentials( + target_credentials=target, + target_audience=audience, + include_email=True, + ) + + from google.oauth2.id_token import fetch_id_token_credentials + + return fetch_id_token_credentials(audience, request=Request()) + + +def _google_grpc_credentials(audience: str) -> Any: + import grpc + from google.auth.transport.grpc import AuthMetadataPlugin + from google.auth.transport.requests import Request + + plugin = AuthMetadataPlugin( + _google_id_token_credentials(audience), + Request(), + default_host=urlparse(audience).netloc, + ) + return grpc.composite_channel_credentials( + grpc.ssl_channel_credentials(), + grpc.metadata_call_credentials(plugin), + ) + + +def _google_http_session(audience: str, headers: dict[str, str]) -> Any: + from google.auth.transport.requests import AuthorizedSession + + session = AuthorizedSession(_google_id_token_credentials(audience)) + session.headers.update(headers) + return session + + +def _workload_identity_audience(provider: str) -> str: + value = provider.strip() + if value.startswith("//iam.googleapis.com/"): + return value + if value.startswith("projects/"): + return f"//iam.googleapis.com/{value}" + return value + + +def _write_subject_token(token: str) -> str: + path = Path(tempfile.gettempdir()) / "policyengine-observability-otel.jwt" + path.write_text(token) + path.chmod(0o600) + return str(path) diff --git a/policyengine_observability/otel.py b/policyengine_observability/otel.py index e94f553..5596b6f 100644 --- a/policyengine_observability/otel.py +++ b/policyengine_observability/otel.py @@ -1,14 +1,12 @@ from __future__ import annotations -import os import threading from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import UTC, datetime from typing import Any, cast -from urllib.parse import urlparse -from .config import ObservabilityConfig, OTelConfig +from .config import ObservabilityConfig, OTLPExporterConfig from .diagnostics import Diagnostics from .schema import metric_attributes @@ -83,7 +81,8 @@ def _configure_owned_providers(self) -> None: resource = Resource.create(self.resource_attributes()) readers: list[Any] = [] otel = self.config.otel - if otel.endpoint and otel.metrics_enabled: + metric_exporter = otel.metrics + if metric_exporter is not None: from opentelemetry.sdk.metrics.export import ( PeriodicExportingMetricReader, ) @@ -91,7 +90,7 @@ def _configure_owned_providers(self) -> None: readers.append( PeriodicExportingMetricReader( _lazy_metric_exporter_class()( - lambda: _build_metric_exporter(otel), + lambda: _build_metric_exporter(metric_exporter), self.diagnostics, ), export_interval_millis=max( @@ -100,7 +99,7 @@ def _configure_owned_providers(self) -> None: ), export_timeout_millis=max( 1, - otel.export_timeout_seconds * 1_000, + metric_exporter.timeout_seconds * 1_000, ), ) ) @@ -120,7 +119,8 @@ def _configure_owned_providers(self) -> None: meter_provider=self._meter_provider, ) self._owns_tracer_provider = True - if otel.endpoint and otel.traces_enabled: + trace_exporter = otel.traces + if trace_exporter is not None: from opentelemetry.sdk.trace.export import BatchSpanProcessor batch_size = min( @@ -132,7 +132,7 @@ def _configure_owned_providers(self) -> None: cast( Any, _LazySpanExporter( - lambda: _build_span_exporter(otel), + lambda: _build_span_exporter(trace_exporter), self.diagnostics, ), ), @@ -144,7 +144,7 @@ def _configure_owned_providers(self) -> None: ), export_timeout_millis=max( 1, - otel.export_timeout_seconds * 1_000, + trace_exporter.timeout_seconds * 1_000, ), meter_provider=self._meter_provider, ) @@ -539,16 +539,14 @@ def _get(self) -> Any: return LazyMetricExporter -def _build_span_exporter(config: OTelConfig) -> Any: +def _build_span_exporter(config: OTLPExporterConfig) -> Any: kwargs = _exporter_kwargs(config) if config.protocol == "http/protobuf": from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( OTLPSpanExporter, ) - kwargs["endpoint"] = _http_signal_endpoint( - config.endpoint or "", "traces" - ) + kwargs["endpoint"] = _http_signal_endpoint(config.endpoint, "traces") return OTLPSpanExporter(**kwargs) from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( OTLPSpanExporter, @@ -557,16 +555,14 @@ def _build_span_exporter(config: OTelConfig) -> Any: return OTLPSpanExporter(**kwargs) -def _build_metric_exporter(config: OTelConfig) -> Any: +def _build_metric_exporter(config: OTLPExporterConfig) -> Any: kwargs = _exporter_kwargs(config) if config.protocol == "http/protobuf": from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( OTLPMetricExporter, ) - kwargs["endpoint"] = _http_signal_endpoint( - config.endpoint or "", "metrics" - ) + kwargs["endpoint"] = _http_signal_endpoint(config.endpoint, "metrics") return OTLPMetricExporter(**kwargs) from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( OTLPMetricExporter, @@ -575,91 +571,25 @@ def _build_metric_exporter(config: OTelConfig) -> Any: return OTLPMetricExporter(**kwargs) -def _exporter_kwargs(config: OTelConfig) -> dict[str, Any]: +def _exporter_kwargs(config: OTLPExporterConfig) -> dict[str, Any]: headers = dict(config.headers) kwargs: dict[str, Any] = { "endpoint": config.endpoint, "headers": headers, - "timeout": max(0.1, min(config.export_timeout_seconds, 60.0)), + "timeout": max(0.1, min(config.timeout_seconds, 60.0)), } - if config.google_audience: - if config.protocol == "grpc": - kwargs["credentials"] = _google_grpc_credentials( - config.google_audience - ) - else: - kwargs["session"] = _google_http_session( - config.google_audience, - headers, + if config.auth is not None: + kwargs.update( + config.auth.exporter_kwargs( + protocol=config.protocol, + headers=headers, ) + ) + if "session" in kwargs: kwargs.pop("headers", None) return kwargs -def _google_id_token_credentials(audience: str) -> Any: - from google.auth.transport.requests import Request - - modal_token = os.getenv("MODAL_IDENTITY_TOKEN") or os.getenv( - "OBSERVABILITY_GOOGLE_OIDC_TOKEN" - ) - provider = os.getenv("OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER") - service_account = os.getenv("OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL") - if modal_token and provider and service_account: - from google.auth import identity_pool, impersonated_credentials - - source = identity_pool.Credentials.from_info( - { - "type": "external_account", - "audience": _workload_identity_audience(provider), - "subject_token_type": ("urn:ietf:params:oauth:token-type:jwt"), - "token_url": "https://sts.googleapis.com/v1/token", - "credential_source": { - "file": _write_subject_token(modal_token), - "format": {"type": "text"}, - }, - }, - scopes=["https://www.googleapis.com/auth/cloud-platform"], - ) - target = impersonated_credentials.Credentials( - source_credentials=source, - target_principal=service_account, - target_scopes=["https://www.googleapis.com/auth/cloud-platform"], - ) - return impersonated_credentials.IDTokenCredentials( - target_credentials=target, - target_audience=audience, - include_email=True, - ) - - from google.oauth2.id_token import fetch_id_token_credentials - - return fetch_id_token_credentials(audience, request=Request()) - - -def _google_grpc_credentials(audience: str) -> Any: - import grpc - from google.auth.transport.grpc import AuthMetadataPlugin - from google.auth.transport.requests import Request - - plugin = AuthMetadataPlugin( - _google_id_token_credentials(audience), - Request(), - default_host=urlparse(audience).netloc, - ) - return grpc.composite_channel_credentials( - grpc.ssl_channel_credentials(), - grpc.metadata_call_credentials(plugin), - ) - - -def _google_http_session(audience: str, headers: Mapping[str, str]) -> Any: - from google.auth.transport.requests import AuthorizedSession - - session = AuthorizedSession(_google_id_token_credentials(audience)) - session.headers.update(headers) - return session - - def _http_signal_endpoint(endpoint: str, signal: str) -> str: value = endpoint.rstrip("/") if value.endswith(f"/v1/{signal}"): @@ -667,27 +597,6 @@ def _http_signal_endpoint(endpoint: str, signal: str) -> str: return f"{value}/v1/{signal}" -def _workload_identity_audience(provider: str) -> str: - value = provider.strip() - if value.startswith("//iam.googleapis.com/"): - return value - if value.startswith("projects/"): - return f"//iam.googleapis.com/{value}" - return value - - -def _write_subject_token(token: str) -> str: - import tempfile - from pathlib import Path - - path = Path(tempfile.gettempdir()) / ( - "policyengine-observability-otel-oidc.jwt" - ) - path.write_text(token) - path.chmod(0o600) - return str(path) - - def captured_at_is_recent(value: Any, max_age_seconds: float) -> bool: if not isinstance(value, str): return False diff --git a/policyengine_observability/runtime.py b/policyengine_observability/runtime.py index 8113286..fc27f83 100644 --- a/policyengine_observability/runtime.py +++ b/policyengine_observability/runtime.py @@ -16,6 +16,7 @@ from .config import ObservabilityConfig from .delivery import DeliveryManager +from .destinations import StdoutLogDestination from .diagnostics import Diagnostics from .otel import OTelRuntime, SpanHandle, captured_at_is_recent from .schema import build_record, normalize_attributes @@ -96,10 +97,8 @@ def _new_delivery(self) -> DeliveryManager: fallback = ObservabilityConfig( service=self.config.service, deployment=self.config.deployment, - google_cloud_project_id=self.config.google_cloud_project_id, logging=self.config.logging.__class__( - stdout_enabled=True, - remote=None, + destinations=(StdoutLogDestination(),), ), otel=self.config.otel, limits=self.config.limits, @@ -125,9 +124,6 @@ def _new_otel(self) -> OTelRuntime: ObservabilityConfig( service=self.config.service, deployment=self.config.deployment, - google_cloud_project_id=( - self.config.google_cloud_project_id - ), logging=self.config.logging, otel=self.config.otel.__class__(enabled=False), limits=self.config.limits, diff --git a/policyengine_observability/schema.py b/policyengine_observability/schema.py index 9e529f2..5ad9877 100644 --- a/policyengine_observability/schema.py +++ b/policyengine_observability/schema.py @@ -109,18 +109,6 @@ def build_record( if error is not None: record.update(_error_fields(error, config)) - trace_id = record.get("trace_id") - span_id = record.get("span_id") - project_id = config.google_cloud_project_id - if trace_id and project_id: - record["logging.googleapis.com/trace"] = ( - f"projects/{project_id}/traces/{trace_id}" - ) - record["logging.googleapis.com/trace_sampled"] = bool( - record.get("trace_sampled", False) - ) - if span_id: - record["logging.googleapis.com/spanId"] = span_id return record diff --git a/tests/conftest.py b/tests/conftest.py index b20f6db..c19a707 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,7 +21,7 @@ def make_config(**overrides): values = { "service": ServiceIdentity( name="test-api", - namespace="policyengine.api-v1", + namespace="policyengine.test", version="2.3.4", role="entry", ), @@ -31,9 +31,12 @@ def make_config(**overrides): region="us-central1", instance_id="instance-1", ), - "google_cloud_project_id": "policyengine-observability", - "logging": LoggingConfig(stdout_enabled=True), + "logging": LoggingConfig(), "otel": OTelConfig(enabled=False), + "application_attribute_keys": frozenset({"auth_result", "backend"}), + "dispatch_attribute_keys": frozenset( + {"job_id", "run_id", "simulation_id"} + ), } values.update(overrides) return ObservabilityConfig(**values) diff --git a/tests/test_config_schema.py b/tests/test_config_schema.py index 8d7557f..634af05 100644 --- a/tests/test_config_schema.py +++ b/tests/test_config_schema.py @@ -7,10 +7,12 @@ from policyengine_observability import ( SCHEMA_VERSION, DeploymentIdentity, - GoogleCloudLoggingConfig, + GoogleCloudLogDestination, + GoogleCloudLogFormatter, LoggingConfig, ObservabilityConfig, OTelConfig, + OTLPExporterConfig, ServiceIdentity, TelemetryLimits, ) @@ -24,7 +26,7 @@ def test_explicit_identity_is_required_by_constructor() -> None: config = make_config() assert config.identity_complete - assert config.service.namespace == "policyengine.api-v1" + assert config.service.namespace == "policyengine.test" assert config.deployment.platform == "local" @@ -33,24 +35,30 @@ def test_incomplete_identity_reports_diagnostics() -> None: service=ServiceIdentity("", "", "", ""), deployment=DeploymentIdentity("", "other"), logging=LoggingConfig( - remote=GoogleCloudLoggingConfig(project_id="", log_name="") + destinations=( + GoogleCloudLogDestination(project_id="", log_name=""), + ) ), - otel=OTelConfig(enabled=True, endpoint=None), + otel=OTelConfig(enabled=True), ) messages = " ".join(config.diagnostics()) assert not config.identity_complete assert "Missing explicit runtime identity" in messages - assert "Remote logging is disabled" in messages - assert "no OTLP endpoint" in messages + assert "Google Cloud logging is disabled" in messages + assert "no trace or metric OTLP endpoint" in messages -def test_remote_logging_is_rejected_outside_modal() -> None: +def test_remote_logging_is_not_selected_from_platform() -> None: config = make_config( logging=LoggingConfig( - remote=GoogleCloudLoggingConfig(project_id="central") + destinations=( + GoogleCloudLogDestination( + project_id="central", log_name="application" + ), + ) ) ) - assert "outside Modal" in " ".join(config.diagnostics()) + assert config.diagnostics() == () def test_from_env_reads_transport_but_not_identity(monkeypatch) -> None: @@ -62,7 +70,7 @@ def test_from_env_reads_transport_but_not_identity(monkeypatch) -> None: monkeypatch.setenv("OTEL_BSP_MAX_QUEUE_SIZE", "12") monkeypatch.setenv("OTEL_BSP_MAX_EXPORT_BATCH_SIZE", "99") monkeypatch.setenv("OTEL_BSP_SCHEDULE_DELAY", "2500") - monkeypatch.setenv("OTEL_BSP_EXPORT_TIMEOUT", "1500") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TIMEOUT", "1500") monkeypatch.setenv("OTEL_METRIC_EXPORT_INTERVAL", "4000") service = ServiceIdentity("svc", "ns", "1", "api") deployment = DeploymentIdentity("prod", "google_cloud_run") @@ -71,16 +79,17 @@ def test_from_env_reads_transport_but_not_identity(monkeypatch) -> None: ) assert config.service is service assert config.deployment is deployment - assert config.google_cloud_project_id is None - assert config.logging.remote is None - assert config.otel.endpoint == "https://collector" - assert config.otel.protocol == "http/protobuf" - assert config.otel.headers == (("x-one", "1"), ("x-two", "2")) + assert config.otel.traces == OTLPExporterConfig( + endpoint="https://collector", + protocol="http/protobuf", + headers=(("x-one", "1"), ("x-two", "2")), + timeout_seconds=1.5, + ) + assert config.otel.metrics == config.otel.traces assert config.otel.sampling_ratio == 0.25 assert config.otel.span_queue_capacity == 12 assert config.otel.span_batch_size == 99 assert config.otel.span_schedule_delay_seconds == 2.5 - assert config.otel.export_timeout_seconds == 1.5 assert config.otel.metric_export_interval_seconds == 4.0 @@ -93,7 +102,8 @@ def test_from_env_invalid_values_are_bounded(monkeypatch) -> None: service=ServiceIdentity("svc", "ns", "1", "api"), deployment=DeploymentIdentity("dev", "local"), ) - assert config.otel.protocol == "grpc" + assert config.otel.traces is None + assert config.otel.metrics is None assert config.otel.sampling_ratio == 1.0 assert config.otel.span_queue_capacity == 2_048 assert not config.otel.enabled @@ -173,7 +183,7 @@ def test_attribute_count_and_string_length_are_bounded() -> None: assert omitted == 1 -def test_google_trace_correlation_uses_central_project() -> None: +def test_google_trace_correlation_is_not_in_canonical_record() -> None: record = build_record( make_config(), severity="INFO", @@ -184,11 +194,15 @@ def test_google_trace_correlation_uses_central_project() -> None: "trace_sampled": True, }, ) - assert record["logging.googleapis.com/trace"] == ( - "projects/policyengine-observability/traces/" + "a" * 32 + assert "logging.googleapis.com/trace" not in record + assert "logging.googleapis.com/spanId" not in record + assert "logging.googleapis.com/trace_sampled" not in record + formatted = GoogleCloudLogFormatter("central-project")(record) + assert formatted["logging.googleapis.com/trace"] == ( + "projects/central-project/traces/" + "a" * 32 ) - assert record["logging.googleapis.com/spanId"] == "b" * 16 - assert record["logging.googleapis.com/trace_sampled"] is True + assert formatted["logging.googleapis.com/spanId"] == "b" * 16 + assert formatted["logging.googleapis.com/trace_sampled"] is True def test_metric_attributes_use_separate_allowlist() -> None: diff --git a/tests/test_delivery.py b/tests/test_delivery.py index 0ab15cf..4e51f4a 100644 --- a/tests/test_delivery.py +++ b/tests/test_delivery.py @@ -8,12 +8,14 @@ from conftest import make_config from policyengine_observability import ( + CustomLogDestination, DeploymentIdentity, - GoogleCloudLoggingConfig, + GoogleCloudLogDestination, LoggingConfig, + StdoutLogDestination, ) -from policyengine_observability.delivery import ( - DeliveryManager, +from policyengine_observability.delivery import DeliveryManager +from policyengine_observability.destinations.google_cloud import ( _GoogleCloudWriter, ) from policyengine_observability.diagnostics import Diagnostics @@ -37,21 +39,25 @@ def close(self) -> None: self.closed = True -def modal_config(**remote_overrides): - remote_values = { - "project_id": "policyengine-observability", - "queue_capacity": 10, - "batch_size": 5, - } - shutdown_timeout_seconds = remote_overrides.pop( - "shutdown_timeout_seconds", 0.2 - ) - remote_values.update(remote_overrides) +def queued_config(writer_factory, **overrides): + queue_capacity = overrides.pop("queue_capacity", 10) + batch_size = overrides.pop("batch_size", 5) + shutdown_timeout_seconds = overrides.pop("shutdown_timeout_seconds", 0.2) + platform = overrides.pop("platform", "modal") + assert not overrides return make_config( - deployment=DeploymentIdentity("test", "modal"), + deployment=DeploymentIdentity("test", platform), logging=LoggingConfig( - stdout_enabled=True, - remote=GoogleCloudLoggingConfig(**remote_values), + destinations=( + StdoutLogDestination(), + CustomLogDestination( + name="recording", + writer_factory=writer_factory, + delivery="queued", + queue_capacity=queue_capacity, + batch_size=batch_size, + ), + ), shutdown_timeout_seconds=shutdown_timeout_seconds, ), ) @@ -66,56 +72,42 @@ def test_stdout_delivery_is_single_line_json() -> None: assert not manager.remote_enabled -def test_modal_delivery_writes_stdout_and_uses_lazy_worker() -> None: +def test_queued_delivery_writes_stdout_and_uses_lazy_worker() -> None: output = io.StringIO() diagnostics = Diagnostics() writer = RecordingWriter() factory_calls = 0 - def factory(_config): + def factory(): nonlocal factory_calls factory_calls += 1 return writer manager = DeliveryManager( - modal_config(), + queued_config(factory), diagnostics, stdout=output, - writer_factory=factory, ) assert factory_calls == 0 - manager.emit({"severity": "INFO", "event.name": "modal"}) - assert json.loads(output.getvalue())["event.name"] == "modal" + manager.emit({"severity": "INFO", "event.name": "queued"}) + assert json.loads(output.getvalue())["event.name"] == "queued" assert writer.written.wait(1) - assert writer.records[0]["event.name"] == "modal" + assert writer.records[0]["event.name"] == "queued" assert factory_calls == 1 manager.close(1) assert writer.closed -def test_cloud_run_never_creates_direct_writer() -> None: - calls = 0 - - def factory(_config): - nonlocal calls - calls += 1 - return RecordingWriter() - +def test_explicit_queued_destination_is_platform_independent() -> None: + writer = RecordingWriter() manager = DeliveryManager( - make_config( - deployment=DeploymentIdentity("prod", "google_cloud_run"), - logging=LoggingConfig( - remote=GoogleCloudLoggingConfig(project_id="central") - ), - ), + queued_config(lambda: writer, platform="google_cloud_run"), Diagnostics(), stdout=io.StringIO(), - writer_factory=factory, ) manager.emit({"event.name": "cloud-run"}) - manager.close() - assert calls == 0 - assert not manager.remote_enabled + assert writer.written.wait(1) + manager.close(1) def test_queue_saturation_drops_without_blocking() -> None: @@ -131,10 +123,13 @@ def write_many(self, records: list[dict]) -> None: writer = BlockingWriter() manager = DeliveryManager( - modal_config(queue_capacity=1, batch_size=1), + queued_config( + lambda: writer, + queue_capacity=1, + batch_size=1, + ), diagnostics, stdout=io.StringIO(), - writer_factory=lambda _config: writer, ) manager.emit({"sequence": 1}) assert started.wait(1) @@ -151,15 +146,14 @@ def test_writer_auth_or_permission_failure_is_nonfatal() -> None: diagnostics = Diagnostics() attempted = threading.Event() - def unavailable(_config): + def unavailable(): attempted.set() raise PermissionError("denied") manager = DeliveryManager( - modal_config(), + queued_config(unavailable), diagnostics, stdout=io.StringIO(), - writer_factory=unavailable, ) manager.emit({"event.name": "survives"}) assert attempted.wait(1) @@ -173,16 +167,14 @@ def unavailable(_config): def test_malformed_record_and_closed_queue_are_nonfatal() -> None: diagnostics = Diagnostics() - output = io.StringIO() - manager = DeliveryManager(make_config(), diagnostics, stdout=output) + manager = DeliveryManager(make_config(), diagnostics, stdout=io.StringIO()) manager.emit({"bad": object()}) - assert diagnostics.count("failure.stdout.write") == 1 + assert diagnostics.count("logs.export_failure") == 1 remote = DeliveryManager( - modal_config(), + queued_config(lambda: RecordingWriter()), diagnostics, stdout=io.StringIO(), - writer_factory=lambda _config: RecordingWriter(), ) remote.close(1) remote.emit({"event.name": "after-close"}) @@ -200,10 +192,12 @@ def write_many(self, records: list[dict]) -> None: release.wait(1) manager = DeliveryManager( - modal_config(shutdown_timeout_seconds=0.01), + queued_config( + lambda: BlockingWriter(), + shutdown_timeout_seconds=0.01, + ), diagnostics, stdout=io.StringIO(), - writer_factory=lambda _config: BlockingWriter(), ) manager.emit({"event.name": "block"}) assert started.wait(1) @@ -242,7 +236,7 @@ def __init__(self, **kwargs): self.closed = False def logger(self, name): - assert name == "policyengine-api-v1-modal" + assert name == "application" return Logger() def close(self): @@ -254,15 +248,19 @@ def close(self): lambda **_kwargs: "credentials", ) writer = _GoogleCloudWriter( - GoogleCloudLoggingConfig(project_id="central", write_timeout_seconds=2) + GoogleCloudLogDestination( + project_id="central", + log_name="application", + write_timeout_seconds=2, + ) ) writer.write_many( [ { "severity": "WARNING", - "logging.googleapis.com/trace": "projects/central/traces/a", - "logging.googleapis.com/spanId": "b", - "logging.googleapis.com/trace_sampled": True, + "trace_id": "a", + "span_id": "b", + "trace_sampled": True, }, {"severity": "INFO"}, ] diff --git a/tests/test_deployment_assets.py b/tests/test_deployment_assets.py deleted file mode 100644 index f74efb1..0000000 --- a/tests/test_deployment_assets.py +++ /dev/null @@ -1,141 +0,0 @@ -from __future__ import annotations - -import json -import os -import stat -import subprocess -import sys -from pathlib import Path - -ROOT = Path(__file__).parents[1] -DEPLOY = ROOT / "deploy" / "gcp" - - -def test_dashboard_is_valid_json_with_required_signals() -> None: - dashboard = json.loads((DEPLOY / "dashboard.template.json").read_text()) - serialized = json.dumps(dashboard) - assert "policyengine.request.count" in serialized - assert "policyengine.request.duration" in serialized - assert "policyengine.error.count" in serialized - assert "policyengine.telemetry.dropped" in serialized - assert "policyengine.telemetry.exporter.failure" in serialized - - -def test_collector_accepts_only_traces_and_metrics() -> None: - config = (DEPLOY / "collector" / "config.yaml").read_text() - assert "telemetry.googleapis.com:443" in config - assert "memory_limiter" in config - assert "googleclientauth" in config - assert " traces:" in config - assert " metrics:" in config - assert " logs:\n receivers:" not in config - - -def test_authorization_assets_exclude_unrelated_applications() -> None: - iam = (DEPLOY / "iam.template.yaml").read_text() - routing = (DEPLOY / "log-routing.template.yaml").read_text() - for excluded in ( - "policyengine-household-api", - "policyengine-uk-chat", - "peukchat", - "precompute", - "smoke", - "ephemeral", - ): - assert excluded not in iam - assert excluded not in routing - assert "policyengine-simulation-gateway" in iam - assert "policyengine-simulation-py" in iam - assert 'jsonPayload."service.namespace"' in routing - - -def test_deployment_templates_use_environment_placeholders() -> None: - templates = [ - DEPLOY / "iam.template.yaml", - DEPLOY / "workload-inventory.template.yaml", - DEPLOY / "log-routing.template.yaml", - DEPLOY / "alerts.template.yaml", - DEPLOY / "dashboard.template.json", - DEPLOY / "collector" / "service.template.yaml", - ] - content = "\n".join(path.read_text() for path in templates) - for variable in ( - "OBSERVABILITY_PROJECT_ID", - "OBSERVABILITY_PROJECT_NUMBER", - "API_PROJECT_ID", - "SIMULATION_ENTRY_PROJECT_ID", - "MODAL_WORKSPACE_ID", - ): - assert f"${{{variable}}}" in content - assert "workspace_id: ac-" not in content - - -def test_deployment_renderer_validates_and_does_not_print_values( - tmp_path: Path, -) -> None: - values = { - "OBSERVABILITY_PROJECT_ID": "central-observability", - "OBSERVABILITY_PROJECT_NUMBER": "123456789012", - "API_PROJECT_ID": "api-project", - "SIMULATION_ENTRY_PROJECT_ID": "simulation-entry-project", - "MODAL_WORKSPACE_ID": "ac-private-workspace", - } - environment = os.environ.copy() - environment.update(values) - result = subprocess.run( - [ - sys.executable, - str(DEPLOY / "render_deployment.py"), - "--output-dir", - str(tmp_path), - ], - check=True, - capture_output=True, - text=True, - env=environment, - ) - - assert all(value not in result.stdout for value in values.values()) - rendered_iam = (tmp_path / "iam.yaml").read_text() - assert "policyengine-otel-collector@central-observability" in rendered_iam - assert 'assertion.workspace_id == "ac-private-workspace"' in rendered_iam - assert stat.S_IMODE((tmp_path / "iam.yaml").stat().st_mode) == 0o600 - json.loads((tmp_path / "dashboard.json").read_text()) - - -def test_deployment_renderer_rejects_missing_values(tmp_path: Path) -> None: - environment = os.environ.copy() - for variable in ( - "OBSERVABILITY_PROJECT_ID", - "OBSERVABILITY_PROJECT_NUMBER", - "API_PROJECT_ID", - "SIMULATION_ENTRY_PROJECT_ID", - "MODAL_WORKSPACE_ID", - ): - environment.pop(variable, None) - - result = subprocess.run( - [ - sys.executable, - str(DEPLOY / "render_deployment.py"), - "--output-dir", - str(tmp_path), - ], - check=False, - capture_output=True, - text=True, - env=environment, - ) - - assert result.returncode != 0 - assert "Missing deployment variables:" in result.stderr - assert not list(tmp_path.iterdir()) - - -def test_verification_script_has_valid_shell_syntax() -> None: - subprocess.run( - ["bash", "-n", str(DEPLOY / "verify.sh")], - check=True, - capture_output=True, - text=True, - ) diff --git a/tests/test_destination_strategies.py b/tests/test_destination_strategies.py new file mode 100644 index 0000000..b9df376 --- /dev/null +++ b/tests/test_destination_strategies.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import io +import json +import threading + +from conftest import make_config + +from policyengine_observability import ( + CustomLogDestination, + DeploymentIdentity, + GoogleCloudLogFormatter, + LoggingConfig, + StdoutLogDestination, +) +from policyengine_observability.delivery import DeliveryManager +from policyengine_observability.diagnostics import Diagnostics + + +class RecordingWriter: + def __init__(self) -> None: + self.records: list[dict] = [] + self.written = threading.Event() + + def write(self, record: dict) -> None: + self.records.append(record) + self.written.set() + + def close(self) -> None: + pass + + +class FailingWriter: + def write(self, _record: dict) -> None: + raise RuntimeError("destination failed") + + +def test_remote_destination_selection_is_independent_of_platform() -> None: + writer = RecordingWriter() + manager = DeliveryManager( + make_config( + deployment=DeploymentIdentity("test", "other"), + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="recording", + writer_factory=lambda: writer, + delivery="queued", + ), + ) + ), + ), + Diagnostics(), + ) + + manager.emit({"event.name": "portable"}) + + assert writer.written.wait(1) + assert writer.records == [{"event.name": "portable"}] + manager.close(1) + + +def test_multiple_runtimes_can_write_to_separate_destinations() -> None: + first = RecordingWriter() + second = RecordingWriter() + + def manager_for(writer: RecordingWriter) -> DeliveryManager: + return DeliveryManager( + make_config( + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="isolated", + writer_factory=lambda: writer, + delivery="queued", + ), + ) + ) + ), + Diagnostics(), + ) + + first_manager = manager_for(first) + second_manager = manager_for(second) + first_manager.emit({"consumer": "first-api"}) + second_manager.emit({"consumer": "household-api"}) + + assert first.written.wait(1) + assert second.written.wait(1) + assert first.records == [{"consumer": "first-api"}] + assert second.records == [{"consumer": "household-api"}] + first_manager.close(1) + second_manager.close(1) + + +def test_google_fields_are_added_by_destination_formatter() -> None: + output = io.StringIO() + manager = DeliveryManager( + make_config( + logging=LoggingConfig( + destinations=( + StdoutLogDestination( + formatter=GoogleCloudLogFormatter("trace-project") + ), + ) + ) + ), + Diagnostics(), + stdout=output, + ) + + manager.emit( + { + "severity": "INFO", + "trace_id": "a" * 32, + "span_id": "b" * 16, + "trace_sampled": True, + } + ) + + record = json.loads(output.getvalue()) + assert record["logging.googleapis.com/trace"] == ( + "projects/trace-project/traces/" + "a" * 32 + ) + assert record["logging.googleapis.com/spanId"] == "b" * 16 + assert record["logging.googleapis.com/trace_sampled"] is True + + +def test_one_destination_failure_does_not_affect_another_destination() -> None: + successful = RecordingWriter() + diagnostics = Diagnostics() + manager = DeliveryManager( + make_config( + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="failing", + writer_factory=FailingWriter, + delivery="inline", + ), + CustomLogDestination( + name="successful", + writer_factory=lambda: successful, + delivery="inline", + ), + ) + ) + ), + diagnostics, + ) + + for sequence in range(4): + manager.emit({"sequence": sequence}) + + assert successful.records == [ + {"sequence": 0}, + {"sequence": 1}, + {"sequence": 2}, + {"sequence": 3}, + ] + assert diagnostics.count("logs.export_failure") == 3 + assert diagnostics.count("failure.logging.destination_disabled") == 1 + + +def test_slow_remote_destination_does_not_delay_another_destination() -> None: + started = threading.Event() + release = threading.Event() + successful = RecordingWriter() + + class BlockingWriter: + def write(self, _record: dict) -> None: + started.set() + release.wait(1) + + manager = DeliveryManager( + make_config( + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="blocking", + writer_factory=BlockingWriter, + ), + CustomLogDestination( + name="successful", + writer_factory=lambda: successful, + ), + ) + ) + ), + Diagnostics(), + ) + + manager.emit({"event.name": "independent"}) + + assert started.wait(1) + assert successful.written.wait(0.2) + release.set() + manager.close(1) diff --git a/tests/test_diagnostics_public_api.py b/tests/test_diagnostics_public_api.py index 19e67e0..da128c2 100644 --- a/tests/test_diagnostics_public_api.py +++ b/tests/test_diagnostics_public_api.py @@ -24,6 +24,7 @@ "end_scope", "set_observability_runtime", "register_destination", + "GoogleCloudLoggingConfig", } @@ -93,7 +94,6 @@ def test_automatic_standard_logging_installation(monkeypatch) -> None: try: config = make_config( logging=observability.LoggingConfig( - stdout_enabled=True, capture_standard_library=True, replace_existing_handlers=True, ) diff --git a/tests/test_google_credentials.py b/tests/test_google_credentials.py index 7d0fb5a..cb85081 100644 --- a/tests/test_google_credentials.py +++ b/tests/test_google_credentials.py @@ -54,11 +54,11 @@ def test_modal_workload_identity_configuration(monkeypatch, tmp_path) -> None: ) monkeypatch.setenv( google_credentials.WORKLOAD_IDENTITY_PROVIDER_ENV, - "projects/123/locations/global/workloadIdentityPools/modal/providers/api-v1", + "projects/123/locations/global/workloadIdentityPools/modal/providers/example", ) monkeypatch.setenv( google_credentials.SERVICE_ACCOUNT_EMAIL_ENV, - "modal-api-v1@central.iam.gserviceaccount.com", + "modal-example@central.iam.gserviceaccount.com", ) monkeypatch.setattr( google_credentials, @@ -70,7 +70,7 @@ def test_modal_workload_identity_configuration(monkeypatch, tmp_path) -> None: ) assert config["audience"].startswith("//iam.googleapis.com/projects/123") assert ( - "modal-api-v1@central.iam.gserviceaccount.com" + "modal-example@central.iam.gserviceaccount.com" in config["service_account_impersonation_url"] ) token_path = tmp_path / "policyengine-observability-oidc.jwt" diff --git a/tests/test_otel.py b/tests/test_otel.py index 03ccefc..6806219 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -13,28 +13,35 @@ InMemorySpanExporter, ) -from policyengine_observability import OTelConfig, configure +from policyengine_observability import ( + GoogleIdTokenAuth, + OTelConfig, + OTLPExporterConfig, + configure, +) from policyengine_observability.diagnostics import Diagnostics +from policyengine_observability.google_auth import ( + _google_grpc_credentials, + _google_http_session, + _google_id_token_credentials, + _workload_identity_audience, + _write_subject_token, +) from policyengine_observability.otel import ( OTelRuntime, SpanHandle, _build_metric_exporter, _build_span_exporter, _exporter_kwargs, - _google_grpc_credentials, - _google_http_session, - _google_id_token_credentials, _http_signal_endpoint, _lazy_metric_exporter_class, _LazySpanExporter, - _workload_identity_audience, - _write_subject_token, captured_at_is_recent, ) def _runtime_with_spans(): - config = make_config(otel=OTelConfig(enabled=True, endpoint=None)) + config = make_config(otel=OTelConfig(enabled=True)) runtime = configure(config) runtime._delivery._stdout = io.StringIO() exporter = InMemorySpanExporter() @@ -80,7 +87,7 @@ def test_incoming_trace_context_correlates_log_and_response() -> None: runtime.end_request(status_code=200) item = records(runtime._delivery._stdout)[0] assert item["trace_id"] == trace_id - assert item["logging.googleapis.com/trace"].endswith(trace_id) + assert "logging.googleapis.com/trace" not in item span = exporter.get_finished_spans()[0] assert span.parent.span_id == int(parent_id, 16) runtime.shutdown() @@ -242,18 +249,18 @@ def constructor(name): monkeypatch.setattr( http_metric, "OTLPMetricExporter", constructor("http-metric") ) - grpc = OTelConfig( + grpc = OTLPExporterConfig( endpoint="collector:4317", protocol="grpc", headers=(("x-test", "value"),), - export_timeout_seconds=2, + timeout_seconds=2, ) assert _build_span_exporter(grpc) == "grpc-span" assert _build_metric_exporter(grpc) == "grpc-metric" - http = OTelConfig( + http = OTLPExporterConfig( endpoint="https://collector/", protocol="http/protobuf", - export_timeout_seconds=3, + timeout_seconds=3, ) assert _build_span_exporter(http) == "http-span" assert _build_metric_exporter(http) == "http-metric" @@ -267,25 +274,25 @@ def test_google_exporter_kwargs_select_protocol_credentials( monkeypatch, ) -> None: monkeypatch.setattr( - "policyengine_observability.otel._google_grpc_credentials", + "policyengine_observability.google_auth._google_grpc_credentials", lambda audience: f"grpc:{audience}", ) monkeypatch.setattr( - "policyengine_observability.otel._google_http_session", + "policyengine_observability.google_auth._google_http_session", lambda audience, headers: (audience, dict(headers)), ) grpc = _exporter_kwargs( - OTelConfig( + OTLPExporterConfig( endpoint="collector:4317", - google_audience="https://collector", + auth=GoogleIdTokenAuth("https://collector"), ) ) assert grpc["credentials"] == "grpc:https://collector" http = _exporter_kwargs( - OTelConfig( + OTLPExporterConfig( endpoint="https://collector", protocol="http/protobuf", - google_audience="https://collector", + auth=GoogleIdTokenAuth("https://collector"), headers=(("x", "y"),), ) ) @@ -318,13 +325,13 @@ def __init__(self, **kwargs): impersonated_credentials, "IDTokenCredentials", Identity ) monkeypatch.setattr( - "policyengine_observability.otel._write_subject_token", + "policyengine_observability.google_auth._write_subject_token", lambda token: f"/tmp/{token}", ) monkeypatch.setenv("MODAL_IDENTITY_TOKEN", "modal-token") monkeypatch.setenv( "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER", - "projects/123/providers/api-v1", + "projects/123/providers/example", ) monkeypatch.setenv( "OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL", @@ -360,7 +367,7 @@ def test_google_transport_helpers(monkeypatch, tmp_path) -> None: from google.auth.transport import requests as google_requests monkeypatch.setattr( - "policyengine_observability.otel._google_id_token_credentials", + "policyengine_observability.google_auth._google_id_token_credentials", lambda audience: f"token:{audience}", ) monkeypatch.setattr( diff --git a/tests/test_otel_destinations.py b/tests/test_otel_destinations.py new file mode 100644 index 0000000..804a06a --- /dev/null +++ b/tests/test_otel_destinations.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from policyengine_observability import ( + DeploymentIdentity, + GoogleIdTokenAuth, + ObservabilityConfig, + OTLPExporterConfig, + ServiceIdentity, +) + + +def test_signal_specific_otlp_environment_configuration(monkeypatch) -> None: + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://common") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "https://traces") + monkeypatch.setenv( + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "https://metrics" + ) + monkeypatch.setenv( + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", "trace-key=trace-value" + ) + monkeypatch.setenv( + "OTEL_EXPORTER_OTLP_METRICS_HEADERS", "metric-key=metric-value" + ) + + config = ObservabilityConfig.from_env( + service=ServiceIdentity("service", "namespace", "1", "api"), + deployment=DeploymentIdentity("test", "other"), + ) + + assert config.otel.traces == OTLPExporterConfig( + endpoint="https://traces", + headers=(("trace-key", "trace-value"),), + ) + assert config.otel.metrics == OTLPExporterConfig( + endpoint="https://metrics", + headers=(("metric-key", "metric-value"),), + ) + + +def test_google_authentication_is_an_explicit_exporter_strategy( + monkeypatch, +) -> None: + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "collector:4317") + monkeypatch.setenv( + "POLICYENGINE_OTEL_GOOGLE_AUDIENCE", "https://collector" + ) + + config = ObservabilityConfig.from_env( + service=ServiceIdentity("service", "namespace", "1", "worker"), + deployment=DeploymentIdentity("test", "modal"), + ) + + assert isinstance(config.otel.traces.auth, GoogleIdTokenAuth) + assert isinstance(config.otel.metrics.auth, GoogleIdTokenAuth) + assert config.otel.traces.auth.audience == "https://collector" diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 379f74f..5d60f32 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -199,7 +199,7 @@ def test_two_runtimes_keep_identity_and_context_separate() -> None: first, first_output = make_runtime() second_config = make_config( service=make_config().service.__class__( - "second-api", "policyengine.api-v1", "1", "worker" + "second-api", "policyengine.test", "1", "worker" ) ) second = configure(second_config) From 46d042538034f2f8e911096f399ba9fb0dc854e3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 22 Sep 2026 20:02:21 +0400 Subject: [PATCH 06/12] Release observability runtime version 2 --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index db4df3a..f92e9f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "policyengine-observability" -version = "1.4.1" +version = "2.0.0" description = "Shared PolicyEngine observability runtime for logs, timings, metrics, and OpenTelemetry." readme = "README.md" authors = [{ name = "PolicyEngine", email = "hello@policyengine.org" }] diff --git a/uv.lock b/uv.lock index a292ab2..70ad15f 100644 --- a/uv.lock +++ b/uv.lock @@ -895,7 +895,7 @@ wheels = [ [[package]] name = "policyengine-observability" -version = "1.4.1" +version = "2.0.0" source = { editable = "." } [package.optional-dependencies] From 4ea849342462eae314bd49978e5c672bf9dcb05b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 22 Sep 2026 20:09:14 +0400 Subject: [PATCH 07/12] Match the logging destination strategy contract --- policyengine_observability/destinations/google_cloud.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/policyengine_observability/destinations/google_cloud.py b/policyengine_observability/destinations/google_cloud.py index bd97313..d9ea828 100644 --- a/policyengine_observability/destinations/google_cloud.py +++ b/policyengine_observability/destinations/google_cloud.py @@ -44,7 +44,8 @@ class GoogleCloudLogDestination: name: str = "google_cloud" delivery: Literal["queued"] = "queued" - def build_writer(self, _context: DestinationBuildContext) -> RecordWriter: + def build_writer(self, context: DestinationBuildContext) -> RecordWriter: + del context if not self.project_id.strip() or not self.log_name.strip(): raise ValueError( "Google Cloud project_id and log_name must be non-empty." From 1d2580d0eeebaa99e6e99fd7cfbf610865ca1341 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 22 Sep 2026 21:39:44 +0400 Subject: [PATCH 08/12] Fail fast on invalid observability configuration --- README.md | 15 +- changelog.d/29.breaking.md | 2 + .../engineering/skills/repository-guidance.md | 5 +- policyengine_observability/__init__.py | 2 + policyengine_observability/config.py | 317 +++++++++++++++--- .../destinations/base.py | 12 + .../destinations/google_cloud.py | 22 +- .../destinations/stdout.py | 5 + policyengine_observability/runtime.py | 26 +- tests/test_config_schema.py | 220 +++++++++++- tests/test_diagnostics_public_api.py | 60 ++++ tests/test_runtime.py | 45 +-- 12 files changed, 620 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index 83a1c32..3aa5210 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,11 @@ config = ObservabilityConfig( runtime = configure(config) ``` +`configure` validates the complete configuration before it creates workers, +exporters, or logging handlers. Invalid values raise `ConfigurationError` with +the fields that must be corrected. Unavailable credentials or destinations +after successful validation remain nonfatal runtime failures. + The default logging destination is one-line JSON on standard output. An OTel runtime without an exporter still creates local trace context for log correlation. It does not send traces or metrics remotely. @@ -294,11 +299,11 @@ def restore_process_state(self): ``` Call `runtime.shutdown()` during orderly process shutdown. Logging and OTel -each use their own timeout. Missing dependencies, invalid configuration, -credential failures, unavailable destinations, queue saturation, exporter -errors, and shutdown timeouts produce rate-limited diagnostics on standard -error. These failures do not change application responses, return values, or -exceptions. +each use their own timeout. After configuration validation succeeds, missing +optional dependencies, credential failures, unavailable destinations, queue +saturation, exporter errors, and shutdown timeouts produce rate-limited +diagnostics on standard error. These runtime failures do not change application +responses, return values, or exceptions. ## Release workflow diff --git a/changelog.d/29.breaking.md b/changelog.d/29.breaking.md index 87102e7..c295444 100644 --- a/changelog.d/29.breaking.md +++ b/changelog.d/29.breaking.md @@ -8,3 +8,5 @@ an independent bounded queue, and logging and OTel failures remain isolated from application responses and calculations. Consumers must replace removed version 1 imports and provide their own identity, destinations, attribute policy, endpoints, and credentials before upgrading. +`configure` validates these values before runtime setup and raises +`ConfigurationError` for invalid configuration. diff --git a/docs/engineering/skills/repository-guidance.md b/docs/engineering/skills/repository-guidance.md index a3bb86c..f17df7c 100644 --- a/docs/engineering/skills/repository-guidance.md +++ b/docs/engineering/skills/repository-guidance.md @@ -57,8 +57,9 @@ uv run --extra dev towncrier check --compare-with origin/main fields in the configured destination formatter or writer. - Keep OTLP transport independent from exporter authentication and allow traces and metrics to use different endpoints. -- Observability failures must fail open: record an internal observability error - when practical, but do not break the application operation being observed. +- Reject invalid configuration before starting runtime components. After + successful validation, record runtime failures internally when practical + without breaking the application operation being observed. - Preserve structured log schemas. Make additive changes when possible; bump schema versions for breaking payload changes. - Keep metric attributes bounded and low-cardinality. Do not put raw paths, diff --git a/policyengine_observability/__init__.py b/policyengine_observability/__init__.py index 224b865..c012bc0 100644 --- a/policyengine_observability/__init__.py +++ b/policyengine_observability/__init__.py @@ -2,6 +2,7 @@ from .adapters import instrument_fastapi, instrument_flask from .config import ( + ConfigurationError, DeploymentIdentity, LoggingConfig, ObservabilityConfig, @@ -38,6 +39,7 @@ "TRACEPARENT_HEADER", "TRACESTATE_HEADER", "CustomLogDestination", + "ConfigurationError", "DeploymentIdentity", "GoogleCloudLogDestination", "GoogleCloudLogFormatter", diff --git a/policyengine_observability/config.py b/policyengine_observability/config.py index c51e150..104abb2 100644 --- a/policyengine_observability/config.py +++ b/policyengine_observability/config.py @@ -1,5 +1,6 @@ from __future__ import annotations +import math import os from dataclasses import dataclass, field from typing import Any, Literal, Protocol, cast @@ -10,6 +11,16 @@ OTLPProtocol = Literal["grpc", "http/protobuf"] ProviderMode = Literal["owned", "external"] + +class ConfigurationError(ValueError): + """Raised before startup when observability configuration is invalid.""" + + def __init__(self, errors: tuple[str, ...]) -> None: + self.errors = errors + details = "\n".join(f"- {error}" for error in errors) + super().__init__(f"Invalid observability configuration:\n{details}") + + DEFAULT_APPLICATION_ATTRIBUTE_KEYS = frozenset(set()) DEFAULT_DISPATCH_ATTRIBUTE_KEYS = frozenset(set()) @@ -135,13 +146,14 @@ def from_env( common_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") common_protocol = _protocol( - os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc") + "OTEL_EXPORTER_OTLP_PROTOCOL", + os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc"), ) common_headers = os.getenv("OTEL_EXPORTER_OTLP_HEADERS", "") common_audience = os.getenv("POLICYENGINE_OTEL_GOOGLE_AUDIENCE") traces = _exporter_from_env( signal="traces", - enabled=os.getenv("OTEL_TRACES_EXPORTER", "otlp") != "none", + enabled=_export_enabled("OTEL_TRACES_EXPORTER"), common_endpoint=common_endpoint, common_protocol=common_protocol, common_headers=common_headers, @@ -149,14 +161,14 @@ def from_env( ) metrics = _exporter_from_env( signal="metrics", - enabled=os.getenv("OTEL_METRICS_EXPORTER", "otlp") != "none", + enabled=_export_enabled("OTEL_METRICS_EXPORTER"), common_endpoint=common_endpoint, common_protocol=common_protocol, common_headers=common_headers, common_audience=common_audience, ) - return cls( + config = cls( service=service, deployment=deployment, logging=logging or LoggingConfig(), @@ -167,26 +179,30 @@ def from_env( provider_mode=_provider_mode( os.getenv("POLICYENGINE_OTEL_PROVIDER_MODE", "owned") ), - sampling_ratio=_bounded_float( + sampling_ratio=_env_float( + "OTEL_TRACES_SAMPLER_ARG", os.getenv("OTEL_TRACES_SAMPLER_ARG"), default=1.0, minimum=0.0, maximum=1.0, ), - span_queue_capacity=_bounded_int( + span_queue_capacity=_env_int( + "OTEL_BSP_MAX_QUEUE_SIZE", os.getenv("OTEL_BSP_MAX_QUEUE_SIZE"), default=2_048, minimum=1, maximum=100_000, ), - span_batch_size=_bounded_int( + span_batch_size=_env_int( + "OTEL_BSP_MAX_EXPORT_BATCH_SIZE", os.getenv("OTEL_BSP_MAX_EXPORT_BATCH_SIZE"), default=512, minimum=1, maximum=10_000, ), span_schedule_delay_seconds=( - _bounded_float( + _env_float( + "OTEL_BSP_SCHEDULE_DELAY", os.getenv("OTEL_BSP_SCHEDULE_DELAY"), default=5_000.0, minimum=1.0, @@ -195,7 +211,8 @@ def from_env( / 1_000 ), metric_export_interval_seconds=( - _bounded_float( + _env_float( + "OTEL_METRIC_EXPORT_INTERVAL", os.getenv("OTEL_METRIC_EXPORT_INTERVAL"), default=60_000.0, minimum=1_000.0, @@ -222,34 +239,171 @@ def from_env( ), sensitive_values=sensitive_values, ) + config.validate() + return config - def diagnostics(self) -> tuple[str, ...]: - messages: list[str] = [] + def validate(self) -> None: + """Reject invalid configuration before runtime setup has side effects.""" + + errors = self.validation_errors() + if errors: + raise ConfigurationError(errors) + + def validation_errors(self) -> tuple[str, ...]: + """Return invalid fields without creating runtime components.""" + + errors: list[str] = [] identity_values = { "service.name": self.service.name, "service.namespace": self.service.namespace, "service.version": self.service.version, "service.role": self.service.role, - "deployment.environment.name": self.deployment.environment, - "cloud.platform": self.deployment.platform, + "deployment.environment": self.deployment.environment, } for key, value in identity_values.items(): - if not str(value).strip(): - messages.append(f"Missing explicit runtime identity: {key}") + if not isinstance(value, str) or not value.strip(): + errors.append(f"{key} must be a non-empty string.") + + _choice_error( + errors, + "deployment.platform", + self.deployment.platform, + {"google_cloud_run", "modal", "local", "other"}, + ) + if not isinstance(self.logging.minimum_severity, int) or isinstance( + self.logging.minimum_severity, bool + ): + errors.append("logging.minimum_severity must be an integer.") + _number_error( + errors, + "logging.shutdown_timeout_seconds", + self.logging.shutdown_timeout_seconds, + 0, + 60, + ) for destination in self.logging.destinations: - validate = getattr(destination, "diagnostics", None) - if not callable(validate): - continue try: - messages.extend( - str(item) for item in cast(tuple[str, ...], validate()) + _choice_error( + errors, + f"logging destination {destination.name!r} delivery", + destination.delivery, + {"inline", "queued"}, + ) + _integer_error( + errors, + f"logging destination {destination.name!r} queue_capacity", + destination.queue_capacity, + 1, + 100_000, + ) + _integer_error( + errors, + f"logging destination {destination.name!r} batch_size", + destination.batch_size, + 1, + 10_000, ) + validate = getattr(destination, "diagnostics", None) + if callable(validate): + errors.extend( + str(item) for item in cast(tuple[str, ...], validate()) + ) except Exception as exc: - messages.append( - "Log destination validation failed for " + errors.append( + "Invalid log destination strategy " f"{getattr(destination, 'name', '')}: {exc}" ) + + _choice_error( + errors, + "otel.provider_mode", + self.otel.provider_mode, + {"owned", "external"}, + ) + _number_error( + errors, "otel.sampling_ratio", self.otel.sampling_ratio, 0, 1 + ) + _integer_error( + errors, + "otel.span_queue_capacity", + self.otel.span_queue_capacity, + 1, + 100_000, + ) + _integer_error( + errors, + "otel.span_batch_size", + self.otel.span_batch_size, + 1, + 10_000, + ) + _number_error( + errors, + "otel.span_schedule_delay_seconds", + self.otel.span_schedule_delay_seconds, + 0.001, + 60, + ) + _number_error( + errors, + "otel.metric_export_interval_seconds", + self.otel.metric_export_interval_seconds, + 1, + 3_600, + ) + _number_error( + errors, + "otel.shutdown_timeout_seconds", + self.otel.shutdown_timeout_seconds, + 0, + 60, + ) + for signal, exporter in ( + ("traces", self.otel.traces), + ("metrics", self.otel.metrics), + ): + if exporter is None: + continue + if ( + not isinstance(exporter.endpoint, str) + or not exporter.endpoint.strip() + ): + errors.append( + f"otel.{signal}.endpoint must be a non-empty string." + ) + _choice_error( + errors, + f"otel.{signal}.protocol", + exporter.protocol, + {"grpc", "http/protobuf"}, + ) + _number_error( + errors, + f"otel.{signal}.timeout_seconds", + exporter.timeout_seconds, + 0.1, + 60, + ) + + for name, value in { + "limits.max_attributes": self.limits.max_attributes, + "limits.max_string_length": self.limits.max_string_length, + "limits.max_error_message_length": self.limits.max_error_message_length, + "limits.max_stack_length": self.limits.max_stack_length, + }.items(): + _integer_error(errors, name, value, 1, 1_000_000) + _number_error( + errors, + "limits.async_parent_max_age_seconds", + self.limits.async_parent_max_age_seconds, + 0, + 86_400, + ) + return tuple(errors) + + def diagnostics(self) -> tuple[str, ...]: + messages: list[str] = [] if ( self.otel.enabled and self.otel.provider_mode == "owned" @@ -281,20 +435,39 @@ def _env_bool(name: str, default: bool) -> bool: value = os.getenv(name) if value is None: return default - return value.strip().lower() in {"1", "true", "yes", "on"} + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + raise ConfigurationError( + (f"{name} must be a boolean value; received {value!r}.",) + ) def _provider_mode(value: str) -> ProviderMode: - return "external" if value.strip().lower() == "external" else "owned" + normalized = value.strip().lower() + if normalized not in {"owned", "external"}: + raise ConfigurationError( + (f"POLICYENGINE_OTEL_PROVIDER_MODE is invalid: {value!r}.",) + ) + return cast(ProviderMode, normalized) -def _protocol(value: str) -> OTLPProtocol: +def _protocol(name: str, value: str) -> OTLPProtocol: normalized = value.strip() if normalized not in {"grpc", "http/protobuf"}: - return "grpc" + raise ConfigurationError((f"{name} is invalid: {value!r}.",)) return cast(OTLPProtocol, normalized) +def _export_enabled(name: str) -> bool: + value = os.getenv(name, "otlp").strip().lower() + if value not in {"otlp", "none"}: + raise ConfigurationError((f"{name} is invalid: {value!r}.",)) + return value == "otlp" + + def _exporter_from_env( *, signal: Literal["traces", "metrics"], @@ -312,13 +485,16 @@ def _exporter_from_env( return None protocol_value = os.getenv(f"{prefix}_PROTOCOL") protocol = ( - _protocol(protocol_value) + _protocol(f"{prefix}_PROTOCOL", protocol_value) if protocol_value is not None else common_protocol ) headers_value = os.getenv(f"{prefix}_HEADERS") headers = _parse_headers( - headers_value if headers_value is not None else common_headers + f"{prefix}_HEADERS" + if headers_value is not None + else "OTEL_EXPORTER_OTLP_HEADERS", + headers_value if headers_value is not None else common_headers, ) audience = ( os.getenv(f"POLICYENGINE_OTEL_{signal.upper()}_GOOGLE_AUDIENCE") @@ -329,9 +505,15 @@ def _exporter_from_env( from .google_auth import GoogleIdTokenAuth auth = GoogleIdTokenAuth(audience) - timeout = _bounded_float( - os.getenv(f"{prefix}_TIMEOUT") - or os.getenv("OTEL_EXPORTER_OTLP_TIMEOUT"), + signal_timeout = os.getenv(f"{prefix}_TIMEOUT") + timeout_name = ( + f"{prefix}_TIMEOUT" + if signal_timeout is not None + else "OTEL_EXPORTER_OTLP_TIMEOUT" + ) + timeout = _env_float( + timeout_name, + signal_timeout or os.getenv("OTEL_EXPORTER_OTLP_TIMEOUT"), default=5_000.0, minimum=100.0, maximum=60_000.0, @@ -345,18 +527,26 @@ def _exporter_from_env( ) -def _parse_headers(value: str) -> tuple[tuple[str, str], ...]: +def _parse_headers(name: str, value: str) -> tuple[tuple[str, str], ...]: headers: list[tuple[str, str]] = [] for item in value.split(","): - if not item.strip() or "=" not in item: + if not item.strip(): continue + if "=" not in item: + raise ConfigurationError( + (f"{name} contains a header without '=': {item!r}.",) + ) key, header_value = item.split("=", 1) - if key.strip(): - headers.append((key.strip(), header_value.strip())) + if not key.strip(): + raise ConfigurationError( + (f"{name} contains an empty header name.",) + ) + headers.append((key.strip(), header_value.strip())) return tuple(headers) -def _bounded_float( +def _env_float( + name: str, value: str | None, *, default: float, @@ -366,13 +556,18 @@ def _bounded_float( try: parsed = float(value) if value is not None else default except (TypeError, ValueError): - return default - if parsed != parsed: - return default - return min(max(parsed, minimum), maximum) + raise ConfigurationError( + (f"{name} must be a number; received {value!r}.",) + ) from None + if not math.isfinite(parsed) or not minimum <= parsed <= maximum: + raise ConfigurationError( + (f"{name} must be between {minimum} and {maximum}.",) + ) + return parsed -def _bounded_int( +def _env_int( + name: str, value: str | None, *, default: int, @@ -382,5 +577,41 @@ def _bounded_int( try: parsed = int(value) if value is not None else default except (TypeError, ValueError): - return default - return min(max(parsed, minimum), maximum) + raise ConfigurationError( + (f"{name} must be an integer; received {value!r}.",) + ) from None + if not minimum <= parsed <= maximum: + raise ConfigurationError( + (f"{name} must be between {minimum} and {maximum}.",) + ) + return parsed + + +def _choice_error( + errors: list[str], name: str, value: Any, choices: set[str] +) -> None: + if value not in choices: + errors.append(f"{name} must be one of: {', '.join(sorted(choices))}.") + + +def _number_error( + errors: list[str], name: str, value: Any, minimum: float, maximum: float +) -> None: + if ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not math.isfinite(value) + or not minimum <= value <= maximum + ): + errors.append(f"{name} must be between {minimum} and {maximum}.") + + +def _integer_error( + errors: list[str], name: str, value: Any, minimum: int, maximum: int +) -> None: + if ( + not isinstance(value, int) + or isinstance(value, bool) + or not minimum <= value <= maximum + ): + errors.append(f"{name} must be between {minimum} and {maximum}.") diff --git a/policyengine_observability/destinations/base.py b/policyengine_observability/destinations/base.py index 3441335..69f2535 100644 --- a/policyengine_observability/destinations/base.py +++ b/policyengine_observability/destinations/base.py @@ -88,5 +88,17 @@ class CustomLogDestination: batch_size: int = 1 formatter: RecordFormatter | None = None + def diagnostics(self) -> tuple[str, ...]: + errors: list[str] = [] + if not self.name.strip(): + errors.append("Custom log destination name must be non-empty.") + if not callable(self.writer_factory): + errors.append( + "Custom log destination writer_factory must be callable." + ) + if self.formatter is not None and not callable(self.formatter): + errors.append("Custom log destination formatter must be callable.") + return tuple(errors) + def build_writer(self, _context: DestinationBuildContext) -> RecordWriter: return _FormattingWriter(self.writer_factory(), self.formatter) diff --git a/policyengine_observability/destinations/google_cloud.py b/policyengine_observability/destinations/google_cloud.py index d9ea828..c251eb3 100644 --- a/policyengine_observability/destinations/google_cloud.py +++ b/policyengine_observability/destinations/google_cloud.py @@ -53,12 +53,22 @@ def build_writer(self, context: DestinationBuildContext) -> RecordWriter: return _GoogleCloudWriter(self) def diagnostics(self) -> tuple[str, ...]: - if self.project_id.strip() and self.log_name.strip(): - return () - return ( - "Google Cloud logging is disabled because project_id or " - "log_name is empty.", - ) + errors: list[str] = [] + if not self.project_id.strip(): + errors.append("Google Cloud logging requires project_id.") + if not self.log_name.strip(): + errors.append("Google Cloud logging requires log_name.") + _timeout = self.write_timeout_seconds + if ( + not isinstance(_timeout, (int, float)) + or isinstance(_timeout, bool) + or not 0.1 <= _timeout <= 60 + ): + errors.append( + "Google Cloud logging write_timeout_seconds must be between " + "0.1 and 60." + ) + return tuple(errors) class _GoogleCloudWriter: diff --git a/policyengine_observability/destinations/stdout.py b/policyengine_observability/destinations/stdout.py index a7347ec..1da438f 100644 --- a/policyengine_observability/destinations/stdout.py +++ b/policyengine_observability/destinations/stdout.py @@ -46,5 +46,10 @@ class StdoutLogDestination: queue_capacity: int = 1 batch_size: int = 1 + def diagnostics(self) -> tuple[str, ...]: + if self.formatter is not None and not callable(self.formatter): + return ("Standard-output formatter must be callable.",) + return () + def build_writer(self, context: DestinationBuildContext) -> RecordWriter: return _StdoutWriter(context, self.formatter) diff --git a/policyengine_observability/runtime.py b/policyengine_observability/runtime.py index fc27f83..9af7fda 100644 --- a/policyengine_observability/runtime.py +++ b/policyengine_observability/runtime.py @@ -64,6 +64,7 @@ class _ChildSpanState: class ObservabilityRuntime: def __init__(self, config: ObservabilityConfig) -> None: + config.validate() self.config = config self.diagnostics = Diagnostics() self._request_state: ContextVar[_RequestState | None] = ContextVar( @@ -83,11 +84,14 @@ def __init__(self, config: ObservabilityConfig) -> None: for warning in config.diagnostics(): self.diagnostics.report("configuration", warning) if config.logging.capture_standard_library: - instrument_logging( - logging.getLogger(), - self, - replace=config.logging.replace_existing_handlers, - ) + try: + instrument_logging( + logging.getLogger(), + self, + replace=config.logging.replace_existing_handlers, + ) + except Exception as exc: + self.diagnostics.report("logging.handler_install", exc) def _new_delivery(self) -> DeliveryManager: try: @@ -802,10 +806,16 @@ def instrument_logging( and handler.runtime is runtime ): return handler - if replace: - logger.handlers.clear() handler = ObservabilityLogHandler(runtime) - logger.addHandler(handler) + previous_handlers = list(logger.handlers) + try: + if replace: + logger.handlers.clear() + logger.addHandler(handler) + except Exception: + if replace: + logger.handlers[:] = previous_handlers + raise runtime._register_logging_handler(logger, handler) return handler diff --git a/tests/test_config_schema.py b/tests/test_config_schema.py index 634af05..04989f2 100644 --- a/tests/test_config_schema.py +++ b/tests/test_config_schema.py @@ -2,10 +2,13 @@ import json +import pytest from conftest import make_config from policyengine_observability import ( SCHEMA_VERSION, + ConfigurationError, + CustomLogDestination, DeploymentIdentity, GoogleCloudLogDestination, GoogleCloudLogFormatter, @@ -14,7 +17,9 @@ OTelConfig, OTLPExporterConfig, ServiceIdentity, + StdoutLogDestination, TelemetryLimits, + configure, ) from policyengine_observability.schema import ( build_record, @@ -30,22 +35,141 @@ def test_explicit_identity_is_required_by_constructor() -> None: assert config.deployment.platform == "local" -def test_incomplete_identity_reports_diagnostics() -> None: +def test_invalid_configuration_fails_before_destination_setup() -> None: + destination_was_built = False + + def build_writer(): + nonlocal destination_was_built + destination_was_built = True + config = make_config( service=ServiceIdentity("", "", "", ""), - deployment=DeploymentIdentity("", "other"), + deployment=DeploymentIdentity("", "invalid"), # type: ignore[arg-type] logging=LoggingConfig( destinations=( GoogleCloudLogDestination(project_id="", log_name=""), - ) + CustomLogDestination( + name="unbuilt", + writer_factory=build_writer, + delivery="inline", + ), + ), + minimum_severity="INVALID", # type: ignore[arg-type] + ), + ) + + with pytest.raises(ConfigurationError) as raised: + configure(config) + + message = str(raised.value) + assert "service.name must be a non-empty string" in message + assert "deployment.platform" in message + assert "logging.minimum_severity" in message + assert "Google Cloud logging requires project_id" in message + assert not destination_was_built + + +def test_missing_otel_exporters_are_a_nonfatal_diagnostic() -> None: + config = make_config(otel=OTelConfig(enabled=True)) + + config.validate() + + assert "no trace or metric OTLP endpoint" in " ".join(config.diagnostics()) + + +def test_invalid_nested_limits_are_reported_together() -> None: + exporter = OTLPExporterConfig( + endpoint="", + protocol="invalid", # type: ignore[arg-type] + timeout_seconds=0, + ) + config = make_config( + logging=LoggingConfig(shutdown_timeout_seconds=float("nan")), + otel=OTelConfig( + traces=exporter, + provider_mode="invalid", # type: ignore[arg-type] + sampling_ratio=2, + span_queue_capacity=0, + span_batch_size=0, + span_schedule_delay_seconds=0, + metric_export_interval_seconds=0, + shutdown_timeout_seconds=100, ), - otel=OTelConfig(enabled=True), + limits=TelemetryLimits( + max_attributes=0, + max_string_length=0, + max_error_message_length=0, + max_stack_length=0, + async_parent_max_age_seconds=-1, + ), + ) + + message = " ".join(config.validation_errors()) + + for field in ( + "logging.shutdown_timeout_seconds", + "otel.provider_mode", + "otel.sampling_ratio", + "otel.span_queue_capacity", + "otel.span_batch_size", + "otel.span_schedule_delay_seconds", + "otel.metric_export_interval_seconds", + "otel.shutdown_timeout_seconds", + "otel.traces.endpoint", + "otel.traces.protocol", + "otel.traces.timeout_seconds", + "limits.max_attributes", + "limits.async_parent_max_age_seconds", + ): + assert field in message + + +def test_invalid_destination_strategies_are_reported() -> None: + class InvalidDestination: + name = "invalid" + delivery = "network" + queue_capacity = 0 + batch_size = 0 + + def diagnostics(self): + raise RuntimeError("validation failed") + + config = make_config( + logging=LoggingConfig( + destinations=( + object(), # type: ignore[arg-type] + InvalidDestination(), # type: ignore[arg-type] + CustomLogDestination( + name="", + writer_factory=None, # type: ignore[arg-type] + formatter=object(), # type: ignore[arg-type] + ), + StdoutLogDestination(formatter=object()), # type: ignore[arg-type] + GoogleCloudLogDestination( + project_id="", + log_name="", + write_timeout_seconds=0, + ), + ) + ) ) - messages = " ".join(config.diagnostics()) - assert not config.identity_complete - assert "Missing explicit runtime identity" in messages - assert "Google Cloud logging is disabled" in messages - assert "no trace or metric OTLP endpoint" in messages + + message = " ".join(config.validation_errors()) + + for expected in ( + "Invalid log destination strategy", + "delivery must be one of", + "queue_capacity", + "batch_size", + "validation failed", + "name must be non-empty", + "writer_factory must be callable", + "formatter must be callable", + "requires project_id", + "requires log_name", + "write_timeout_seconds", + ): + assert expected in message def test_remote_logging_is_not_selected_from_platform() -> None: @@ -65,7 +189,7 @@ def test_from_env_reads_transport_but_not_identity(monkeypatch) -> None: monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "ambient-project") monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://collector") monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf") - monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "x-one=1,bad,x-two=2") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "x-one=1,x-two=2") monkeypatch.setenv("OTEL_TRACES_SAMPLER_ARG", "0.25") monkeypatch.setenv("OTEL_BSP_MAX_QUEUE_SIZE", "12") monkeypatch.setenv("OTEL_BSP_MAX_EXPORT_BATCH_SIZE", "99") @@ -93,20 +217,80 @@ def test_from_env_reads_transport_but_not_identity(monkeypatch) -> None: assert config.otel.metric_export_interval_seconds == 4.0 -def test_from_env_invalid_values_are_bounded(monkeypatch) -> None: - monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "invalid") - monkeypatch.setenv("OTEL_TRACES_SAMPLER_ARG", "nan") - monkeypatch.setenv("OTEL_BSP_MAX_QUEUE_SIZE", "bad") - monkeypatch.setenv("OTEL_SDK_DISABLED", "yes") +@pytest.mark.parametrize( + ("name", "value"), + [ + ("OTEL_EXPORTER_OTLP_PROTOCOL", "invalid"), + ("OTEL_TRACES_SAMPLER_ARG", "nan"), + ("OTEL_BSP_MAX_QUEUE_SIZE", "bad"), + ("POLICYENGINE_OTEL_PROVIDER_MODE", "invalid"), + ("OTEL_TRACES_EXPORTER", "console"), + ("OTEL_SDK_DISABLED", "sometimes"), + ], +) +def test_from_env_rejects_invalid_values(monkeypatch, name, value) -> None: + monkeypatch.setenv(name, value) + + with pytest.raises(ConfigurationError, match=name): + ObservabilityConfig.from_env( + service=ServiceIdentity("svc", "ns", "1", "api"), + deployment=DeploymentIdentity("dev", "local"), + ) + + +def test_from_env_rejects_malformed_headers(monkeypatch) -> None: + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "collector:4317") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "missing-equals") + + with pytest.raises(ConfigurationError, match="without '='"): + ObservabilityConfig.from_env( + service=ServiceIdentity("svc", "ns", "1", "api"), + deployment=DeploymentIdentity("dev", "local"), + ) + + +def test_from_env_accepts_false_boolean_and_disabled_exporters( + monkeypatch, +) -> None: + monkeypatch.setenv("OTEL_SDK_DISABLED", "no") + monkeypatch.setenv("OTEL_TRACES_EXPORTER", "none") + monkeypatch.setenv("OTEL_METRICS_EXPORTER", "none") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "collector:4317") + config = ObservabilityConfig.from_env( service=ServiceIdentity("svc", "ns", "1", "api"), deployment=DeploymentIdentity("dev", "local"), ) + + assert config.otel.enabled assert config.otel.traces is None assert config.otel.metrics is None - assert config.otel.sampling_ratio == 1.0 - assert config.otel.span_queue_capacity == 2_048 - assert not config.otel.enabled + + +@pytest.mark.parametrize( + ("name", "value", "message"), + [ + ( + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", + "invalid", + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", + ), + ("OTEL_EXPORTER_OTLP_HEADERS", "=value", "empty header name"), + ("OTEL_EXPORTER_OTLP_TIMEOUT", "50", "must be between"), + ("OTEL_BSP_MAX_QUEUE_SIZE", "0", "must be between"), + ], +) +def test_from_env_rejects_invalid_exporter_values( + monkeypatch, name, value, message +) -> None: + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "collector:4317") + monkeypatch.setenv(name, value) + + with pytest.raises(ConfigurationError, match=message): + ObservabilityConfig.from_env( + service=ServiceIdentity("svc", "ns", "1", "api"), + deployment=DeploymentIdentity("dev", "local"), + ) def test_schema_preserves_core_fields_and_namespaces_attributes() -> None: diff --git a/tests/test_diagnostics_public_api.py b/tests/test_diagnostics_public_api.py index da128c2..b8c2995 100644 --- a/tests/test_diagnostics_public_api.py +++ b/tests/test_diagnostics_public_api.py @@ -6,6 +6,7 @@ import logging import sys +import pytest from conftest import make_config, make_runtime, records import policyengine_observability as observability @@ -107,6 +108,65 @@ def test_automatic_standard_logging_installation(monkeypatch) -> None: logger.handlers[:] = original_handlers +def test_invalid_automatic_logging_config_preserves_existing_handlers() -> ( + None +): + logger = logging.getLogger() + original_handlers = list(logger.handlers) + existing = logging.StreamHandler(io.StringIO()) + logger.handlers[:] = [existing] + try: + config = make_config( + logging=observability.LoggingConfig( + capture_standard_library=True, + replace_existing_handlers=True, + minimum_severity="INVALID", # type: ignore[arg-type] + ) + ) + + with pytest.raises( + observability.ConfigurationError, + match="logging.minimum_severity", + ): + observability.configure(config) + + assert logger.handlers == [existing] + finally: + logger.handlers[:] = original_handlers + + +def test_automatic_logging_installation_failure_is_nonfatal( + monkeypatch, +) -> None: + logger = logging.getLogger() + original_handlers = list(logger.handlers) + existing = logging.StreamHandler(io.StringIO()) + logger.handlers[:] = [existing] + + def fail_add_handler(_handler) -> None: + raise RuntimeError("logging subsystem rejected handler") + + try: + with monkeypatch.context() as logging_failure: + logging_failure.setattr(logger, "addHandler", fail_add_handler) + runtime = observability.configure( + make_config( + logging=observability.LoggingConfig( + capture_standard_library=True, + replace_existing_handlers=True, + ) + ) + ) + + assert logger.handlers == [existing] + assert ( + runtime.diagnostics.count("failure.logging.handler_install") == 1 + ) + runtime.shutdown() + finally: + logger.handlers[:] = original_handlers + + def test_replace_logging_handlers_and_shutdown_removes_owned_handler() -> None: runtime, _ = make_runtime() logger = logging.getLogger("tests.replace") diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 5d60f32..49db755 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -13,6 +13,7 @@ from policyengine_observability import ( REQUEST_ID_HEADER, + ConfigurationError, LoggingConfig, OTelConfig, configure, @@ -347,42 +348,28 @@ def fail_bounded_shutdown(*_args, **_kwargs) -> None: @pytest.mark.parametrize( - ("logging_timeout", "otel_timeout", "expected_logging", "expected_otel"), + ("logging_timeout", "otel_timeout"), [ - ("invalid", float("inf"), 2.0, 3.0), - (-1.0, 100.0, 0.0, 60.0), + ("invalid", float("inf")), + (-1.0, 100.0), ], ) -def test_shutdown_timeout_configuration_is_safely_bounded( +def test_invalid_shutdown_timeout_configuration_is_rejected( logging_timeout, otel_timeout, - expected_logging, - expected_otel, ) -> None: - observed, _output = make_runtime( - logging=LoggingConfig(shutdown_timeout_seconds=logging_timeout), - otel=OTelConfig( - enabled=False, - shutdown_timeout_seconds=otel_timeout, - ), - ) - observed_timeouts: list[tuple[str, float]] = [] - - class OTel: - def shutdown(self, timeout: float) -> None: - observed_timeouts.append(("otel", timeout)) - - observed._delivery.close = lambda timeout: observed_timeouts.append( - ("logging", timeout) - ) - observed._otel = OTel() - - observed.shutdown() + with pytest.raises(ConfigurationError) as raised: + make_runtime( + logging=LoggingConfig(shutdown_timeout_seconds=logging_timeout), + otel=OTelConfig( + enabled=False, + shutdown_timeout_seconds=otel_timeout, + ), + ) - assert observed_timeouts == [ - ("logging", expected_logging), - ("otel", expected_otel), - ] + message = str(raised.value) + assert "logging.shutdown_timeout_seconds" in message + assert "otel.shutdown_timeout_seconds" in message @pytest.mark.skipif(not hasattr(os, "fork"), reason="requires POSIX fork") From 7f46614c5740dbd50ac49a8930a0c96d518f08ca Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:02:20 +0400 Subject: [PATCH 09/12] Bound observability cleanup and keep credentials in memory --- README.md | 10 +- changelog.d/29.breaking.md | 4 + policyengine_observability/delivery.py | 63 +++++++- policyengine_observability/google_auth.py | 62 ++++---- .../google_credentials.py | 140 ++++++++---------- tests/test_delivery.py | 111 ++++++++++++++ tests/test_google_credentials.py | 77 +++++----- tests/test_otel.py | 22 +-- 8 files changed, 313 insertions(+), 176 deletions(-) diff --git a/README.md b/README.md index 3aa5210..8aea30a 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,9 @@ logging = LoggingConfig( The writer implements `write(record)`. It may optionally implement `write_many(records)` and `close()`. A separate integration package can also provide a class implementing `LogDestinationStrategy`. Network writers should -always use `delivery="queued"`. +always use `delivery="queued"`. Cleanup for inline writers runs on daemon +threads, and orderly shutdown waits for it only within the configured logging +shutdown timeout. ## OTLP destinations and authentication @@ -182,6 +184,12 @@ OTEL_EXPORTER_OTLP_METRICS_HEADERS=x-api-key=metric-key signals. `POLICYENGINE_OTEL_TRACES_GOOGLE_AUDIENCE` and `POLICYENGINE_OTEL_METRICS_GOOGLE_AUDIENCE` override it per signal. +Google authentication constructed from `GCP_CREDENTIALS_JSON`, +`MODAL_IDENTITY_TOKEN`, or `OBSERVABILITY_GOOGLE_OIDC_TOKEN` stays in process +memory. The package passes credential data and subject tokens directly to +Google Auth and does not create temporary credential files. A path explicitly +provided through `GOOGLE_APPLICATION_CREDENTIALS` remains supported. + Applications can share a collector by configuring the same endpoint and credentials. Another application can use a separate collector or any OTLP-compatible service without changing this package. diff --git a/changelog.d/29.breaking.md b/changelog.d/29.breaking.md index c295444..c5a77dd 100644 --- a/changelog.d/29.breaking.md +++ b/changelog.d/29.breaking.md @@ -10,3 +10,7 @@ version 1 imports and provide their own identity, destinations, attribute policy, endpoints, and credentials before upgrading. `configure` validates these values before runtime setup and raises `ConfigurationError` for invalid configuration. +Inline writer cleanup now runs outside application execution and respects the +configured logging shutdown timeout. Google credential JSON and workload +identity subject tokens are passed to Google Auth in memory instead of being +written to temporary files. diff --git a/policyengine_observability/delivery.py b/policyengine_observability/delivery.py index 3da0e14..377562d 100644 --- a/policyengine_observability/delivery.py +++ b/policyengine_observability/delivery.py @@ -26,6 +26,13 @@ class _InlineDestination: failures: int = 0 +@dataclass(slots=True) +class _WriterCleanup: + name: str + thread: threading.Thread + timeout_reported: bool = False + + class DeliveryManager: """Fan out provider-neutral records to independently isolated writers.""" @@ -41,6 +48,8 @@ def __init__( self._stdout = stdout or sys.stdout self._inline: list[_InlineDestination] = [] self._queued: list[_QueuedWriter] = [] + self._cleanup_lock = threading.Lock() + self._cleanup_tasks: list[_WriterCleanup] = [] self._configure() def _configure(self) -> None: @@ -114,7 +123,10 @@ def _disable_inline(self, destination: _InlineDestination) -> None: self._inline.remove(destination) except ValueError: return - self._close_writer(destination.writer, destination.strategy.name) + self._start_writer_cleanup( + destination.writer, + destination.strategy.name, + ) self.diagnostics.report( "logging.destination_disabled", RuntimeError( @@ -144,20 +156,57 @@ def close(self, timeout_seconds: float | None = None) -> None: except (TypeError, ValueError): timeout = 2.0 deadline = time.monotonic() + timeout - for destination in tuple(self._queued): - destination.close(max(0.0, deadline - time.monotonic())) - for destination in tuple(self._inline): - self._close_writer( + inline = tuple(self._inline) + self._inline.clear() + for destination in inline: + self._start_writer_cleanup( destination.writer, destination.strategy.name, ) + for destination in tuple(self._queued): + destination.close(max(0.0, deadline - time.monotonic())) + with self._cleanup_lock: + cleanup_tasks = tuple(self._cleanup_tasks) + for task in cleanup_tasks: + task.thread.join(max(0.0, deadline - time.monotonic())) + should_report_timeout = False + with self._cleanup_lock: + if task.thread.is_alive() and not task.timeout_reported: + task.timeout_reported = True + should_report_timeout = True + if should_report_timeout: + self.diagnostics.increment("logs.shutdown_timeout") + self.diagnostics.report( + "logging.shutdown_timeout", + "Log writer cleanup did not stop before its deadline.", + destination=task.name, + ) - def _close_writer(self, writer: RecordWriter, name: str) -> None: + def _start_writer_cleanup(self, writer: RecordWriter, name: str) -> None: close = getattr(writer, "close", None) if not callable(close): return + + def run() -> None: + try: + close() + except Exception as exc: + self.diagnostics.report( + "logging.writer_close", exc, destination=name + ) + + task = _WriterCleanup( + name=name, + thread=threading.Thread( + target=run, + name=f"policyengine-observability-close-{name}", + daemon=True, + ), + ) try: - close() + with self._cleanup_lock: + task.thread.start() + self._cleanup_tasks.append(task) except Exception as exc: self.diagnostics.report( "logging.writer_close", exc, destination=name diff --git a/policyengine_observability/google_auth.py b/policyengine_observability/google_auth.py index de3148b..0f5003e 100644 --- a/policyengine_observability/google_auth.py +++ b/policyengine_observability/google_auth.py @@ -1,13 +1,22 @@ from __future__ import annotations import os -import tempfile from dataclasses import dataclass -from pathlib import Path from typing import Any from urllib.parse import urlparse from .config import OTLPProtocol +from .google_credentials import ( + DEFAULT_STS_TOKEN_URL, + GOOGLE_CREDENTIAL_SCOPES, + JWT_SUBJECT_TOKEN_TYPE, + MODAL_IDENTITY_TOKEN_ENV, + OIDC_TOKEN_ENV, + SERVICE_ACCOUNT_EMAIL_ENV, + WORKLOAD_IDENTITY_PROVIDER_ENV, + _EnvironmentSubjectTokenSupplier, + _workload_identity_audience, +) @dataclass(frozen=True, slots=True) @@ -30,31 +39,26 @@ def exporter_kwargs( def _google_id_token_credentials(audience: str) -> Any: from google.auth.transport.requests import Request - modal_token = os.getenv("MODAL_IDENTITY_TOKEN") or os.getenv( - "OBSERVABILITY_GOOGLE_OIDC_TOKEN" - ) - provider = os.getenv("OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER") - service_account = os.getenv("OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL") - if modal_token and provider and service_account: + token_env_names = (MODAL_IDENTITY_TOKEN_ENV, OIDC_TOKEN_ENV) + has_subject_token = any(os.getenv(name) for name in token_env_names) + provider = os.getenv(WORKLOAD_IDENTITY_PROVIDER_ENV) + service_account = os.getenv(SERVICE_ACCOUNT_EMAIL_ENV) + if has_subject_token and provider and service_account: from google.auth import identity_pool, impersonated_credentials - source = identity_pool.Credentials.from_info( - { - "type": "external_account", - "audience": _workload_identity_audience(provider), - "subject_token_type": ("urn:ietf:params:oauth:token-type:jwt"), - "token_url": "https://sts.googleapis.com/v1/token", - "credential_source": { - "file": _write_subject_token(modal_token), - "format": {"type": "text"}, - }, - }, - scopes=["https://www.googleapis.com/auth/cloud-platform"], + source = identity_pool.Credentials( + audience=_workload_identity_audience(provider), + subject_token_type=JWT_SUBJECT_TOKEN_TYPE, + token_url=DEFAULT_STS_TOKEN_URL, + subject_token_supplier=_EnvironmentSubjectTokenSupplier( + token_env_names + ), + scopes=list(GOOGLE_CREDENTIAL_SCOPES), ) target = impersonated_credentials.Credentials( source_credentials=source, target_principal=service_account, - target_scopes=["https://www.googleapis.com/auth/cloud-platform"], + target_scopes=list(GOOGLE_CREDENTIAL_SCOPES), ) return impersonated_credentials.IDTokenCredentials( target_credentials=target, @@ -89,19 +93,3 @@ def _google_http_session(audience: str, headers: dict[str, str]) -> Any: session = AuthorizedSession(_google_id_token_credentials(audience)) session.headers.update(headers) return session - - -def _workload_identity_audience(provider: str) -> str: - value = provider.strip() - if value.startswith("//iam.googleapis.com/"): - return value - if value.startswith("projects/"): - return f"//iam.googleapis.com/{value}" - return value - - -def _write_subject_token(token: str) -> str: - path = Path(tempfile.gettempdir()) / "policyengine-observability-otel.jwt" - path.write_text(token) - path.chmod(0o600) - return str(path) diff --git a/policyengine_observability/google_credentials.py b/policyengine_observability/google_credentials.py index 50efb43..26ea344 100644 --- a/policyengine_observability/google_credentials.py +++ b/policyengine_observability/google_credentials.py @@ -2,7 +2,7 @@ import json import os -import tempfile +from dataclasses import dataclass from pathlib import Path from typing import Any @@ -18,78 +18,87 @@ GOOGLE_CREDENTIAL_SCOPES = ("https://www.googleapis.com/auth/cloud-platform",) +@dataclass(frozen=True, slots=True) +class _EnvironmentSubjectTokenSupplier: + env_names: tuple[str, ...] + + def get_subject_token(self, _context: Any, _request: Any) -> str: + for name in self.env_names: + token = os.getenv(name) + if token: + return token + from google.auth.exceptions import RefreshError + + names = ", ".join(self.env_names) + raise RefreshError(f"No workload identity token found in {names}.") + + def load_google_credentials( *, credentials_json_env: str = "GCP_CREDENTIALS_JSON", application_credentials_env: str = "GOOGLE_APPLICATION_CREDENTIALS", - credentials_path: Path | None = None, prefer_workload_identity: bool = False, ) -> Any | None: """Load Google credentials without performing a network request.""" try: - path: Path | None = None if prefer_workload_identity: - path = _materialize_workload_identity_credentials() - if path is None: - configured_path = os.getenv(application_credentials_env) - if configured_path: - path = Path(configured_path) - if path is None: - path = _materialize_json_credentials( - credentials_json_env=credentials_json_env, - credentials_path=credentials_path, - ) - if path is None and not prefer_workload_identity: - path = _materialize_workload_identity_credentials() - if path is None: - return None - return _load_credentials_from_file(path) + credentials = _workload_identity_credentials() + if credentials is not None: + return credentials + + configured_path = os.getenv(application_credentials_env) + if configured_path: + return _load_credentials_from_file(Path(configured_path)) + + credentials_json = os.getenv(credentials_json_env) + if credentials_json: + return _load_credentials_from_json(credentials_json) + + if not prefer_workload_identity: + return _workload_identity_credentials() + return None except Exception: return None -def _materialize_json_credentials( - *, - credentials_json_env: str, - credentials_path: Path | None, -) -> Path | None: - credentials_json = os.getenv(credentials_json_env) - if not credentials_json: - return None - json.loads(credentials_json) - path = credentials_path or Path(tempfile.gettempdir()).joinpath( - "policyengine-observability-gcp.json" +def _load_credentials_from_json(credentials_json: str) -> Any: + import google.auth + + config = json.loads(credentials_json) + credentials, _project = google.auth.load_credentials_from_dict( + config, + scopes=list(GOOGLE_CREDENTIAL_SCOPES), ) - path.write_text(credentials_json) - path.chmod(0o600) - return path + return credentials -def _materialize_workload_identity_credentials() -> Path | None: - token = os.getenv(OIDC_TOKEN_ENV) or os.getenv(MODAL_IDENTITY_TOKEN_ENV) +def _workload_identity_credentials() -> Any | None: + token_env_names = (OIDC_TOKEN_ENV, MODAL_IDENTITY_TOKEN_ENV) + if not any(os.getenv(name) for name in token_env_names): + return None provider = os.getenv(WORKLOAD_IDENTITY_PROVIDER_ENV) - if not token or not provider: + if not provider: return None - directory = Path(tempfile.gettempdir()) - token_path = directory / "policyengine-observability-oidc.jwt" - config_path = directory / "policyengine-observability-wif.json" - token_path.write_text(token) - token_path.chmod(0o600) - config_path.write_text( - json.dumps( - _external_account_config( - provider=provider, - token_path=token_path, - service_account_email=os.getenv(SERVICE_ACCOUNT_EMAIL_ENV), - token_url=( - os.getenv(STS_TOKEN_URL_ENV) or DEFAULT_STS_TOKEN_URL - ), - ) + + from google.auth import identity_pool + + kwargs: dict[str, Any] = { + "audience": _workload_identity_audience(provider), + "subject_token_type": JWT_SUBJECT_TOKEN_TYPE, + "token_url": os.getenv(STS_TOKEN_URL_ENV) or DEFAULT_STS_TOKEN_URL, + "subject_token_supplier": _EnvironmentSubjectTokenSupplier( + token_env_names + ), + "scopes": list(GOOGLE_CREDENTIAL_SCOPES), + } + service_account_email = os.getenv(SERVICE_ACCOUNT_EMAIL_ENV) + if service_account_email: + kwargs["service_account_impersonation_url"] = ( + "https://iamcredentials.googleapis.com/v1/projects/-/" + f"serviceAccounts/{service_account_email}:generateAccessToken" ) - ) - config_path.chmod(0o600) - return config_path + return identity_pool.Credentials(**kwargs) def _load_credentials_from_file(path: Path) -> Any: @@ -112,31 +121,6 @@ def _load_credentials_from_file(path: Path) -> Any: return credentials -def _external_account_config( - *, - provider: str, - token_path: Path, - service_account_email: str | None, - token_url: str, -) -> dict[str, object]: - config: dict[str, object] = { - "type": "external_account", - "audience": _workload_identity_audience(provider), - "subject_token_type": JWT_SUBJECT_TOKEN_TYPE, - "token_url": token_url, - "credential_source": { - "file": str(token_path), - "format": {"type": "text"}, - }, - } - if service_account_email: - config["service_account_impersonation_url"] = ( - "https://iamcredentials.googleapis.com/v1/projects/-/" - f"serviceAccounts/{service_account_email}:generateAccessToken" - ) - return config - - def _workload_identity_audience(provider: str) -> str: value = provider.strip() if value.startswith("//iam.googleapis.com/"): diff --git a/tests/test_delivery.py b/tests/test_delivery.py index 4e51f4a..dca6368 100644 --- a/tests/test_delivery.py +++ b/tests/test_delivery.py @@ -209,6 +209,117 @@ def write_many(self, records: list[dict]) -> None: release.set() +def test_inline_failure_cleanup_does_not_block_emit() -> None: + close_started = threading.Event() + release = threading.Event() + + class FailingWriter: + def write(self, _record: dict) -> None: + raise RuntimeError("write failed") + + def close(self) -> None: + close_started.set() + release.wait(1) + + manager = DeliveryManager( + make_config( + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="failing-inline", + writer_factory=FailingWriter, + delivery="inline", + ), + ), + ) + ), + Diagnostics(stderr=io.StringIO()), + stdout=io.StringIO(), + ) + + manager.emit({"sequence": 1}) + manager.emit({"sequence": 2}) + before = time.perf_counter() + manager.emit({"sequence": 3}) + + assert time.perf_counter() - before < 0.1 + assert close_started.wait(1) + release.set() + manager.close(1) + + +def test_inline_shutdown_cleanup_is_bounded_and_repeatable() -> None: + close_started = threading.Event() + release = threading.Event() + diagnostics = Diagnostics(stderr=io.StringIO()) + close_calls = 0 + + class BlockingCloseWriter(RecordingWriter): + def close(self) -> None: + nonlocal close_calls + close_calls += 1 + close_started.set() + release.wait(1) + + manager = DeliveryManager( + make_config( + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="blocking-inline", + writer_factory=BlockingCloseWriter, + delivery="inline", + ), + ), + shutdown_timeout_seconds=0.01, + ) + ), + diagnostics, + stdout=io.StringIO(), + ) + + before = time.perf_counter() + manager.close(0.01) + manager.close(0.01) + + assert time.perf_counter() - before < 0.2 + assert close_started.is_set() + assert close_calls == 1 + assert diagnostics.count("logs.shutdown_timeout") == 1 + release.set() + + +def test_inline_cleanup_failure_is_contained() -> None: + close_attempted = threading.Event() + diagnostics = Diagnostics(stderr=io.StringIO()) + + class FailingCloseWriter(RecordingWriter): + def close(self) -> None: + close_attempted.set() + raise RuntimeError("close failed") + + manager = DeliveryManager( + make_config( + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="failing-close", + writer_factory=FailingCloseWriter, + delivery="inline", + ), + ), + ) + ), + diagnostics, + stdout=io.StringIO(), + ) + + manager.close(1) + + assert close_attempted.is_set() + assert diagnostics.count("failure.logging.writer_close") == 1 + + def test_google_writer_batches_with_bounded_api_calls(monkeypatch) -> None: from google.cloud import logging_v2 diff --git a/tests/test_google_credentials.py b/tests/test_google_credentials.py index cb85081..29a137b 100644 --- a/tests/test_google_credentials.py +++ b/tests/test_google_credentials.py @@ -1,14 +1,11 @@ from __future__ import annotations import json -import stat from policyengine_observability import google_credentials -def test_missing_and_malformed_credentials_are_nonfatal( - monkeypatch, tmp_path -) -> None: +def test_missing_and_malformed_credentials_are_nonfatal(monkeypatch) -> None: for name in ( "GCP_CREDENTIALS_JSON", "GOOGLE_APPLICATION_CREDENTIALS", @@ -19,36 +16,38 @@ def test_missing_and_malformed_credentials_are_nonfatal( monkeypatch.delenv(name, raising=False) assert google_credentials.load_google_credentials() is None monkeypatch.setenv("GCP_CREDENTIALS_JSON", "not-json") - assert ( - google_credentials.load_google_credentials( - credentials_path=tmp_path / "credentials.json" - ) - is None - ) + assert google_credentials.load_google_credentials() is None + +def test_json_credentials_are_loaded_in_memory(monkeypatch) -> None: + import google.auth -def test_json_credentials_are_materialized_with_private_permissions( - monkeypatch, tmp_path -) -> None: - path = tmp_path / "credentials.json" + config = {"type": "external_account", "audience": "test"} monkeypatch.setenv( "GCP_CREDENTIALS_JSON", - json.dumps({"type": "external_account", "audience": "test"}), + json.dumps(config), ) - monkeypatch.setattr( - google_credentials, - "_load_credentials_from_file", - lambda loaded_path: loaded_path, - ) - result = google_credentials.load_google_credentials(credentials_path=path) - assert result == path - assert stat.S_IMODE(path.stat().st_mode) == 0o600 + calls: dict[str, object] = {} + marker = object() + def load_credentials_from_dict(info, *, scopes): + calls["info"] = info + calls["scopes"] = scopes + return marker, "project" -def test_modal_workload_identity_configuration(monkeypatch, tmp_path) -> None: monkeypatch.setattr( - google_credentials.tempfile, "gettempdir", lambda: str(tmp_path) + google.auth, + "load_credentials_from_dict", + load_credentials_from_dict, ) + assert google_credentials.load_google_credentials() is marker + assert calls["info"] == config + assert calls["scopes"] == list(google_credentials.GOOGLE_CREDENTIAL_SCOPES) + + +def test_modal_workload_identity_configuration(monkeypatch) -> None: + from google.auth import identity_pool + monkeypatch.setenv( google_credentials.MODAL_IDENTITY_TOKEN_ENV, "signed-test-token" ) @@ -60,22 +59,26 @@ def test_modal_workload_identity_configuration(monkeypatch, tmp_path) -> None: google_credentials.SERVICE_ACCOUNT_EMAIL_ENV, "modal-example@central.iam.gserviceaccount.com", ) - monkeypatch.setattr( - google_credentials, - "_load_credentials_from_file", - lambda path: json.loads(path.read_text()), - ) + calls: dict[str, object] = {} + + def credentials(**kwargs): + calls.update(kwargs) + return "credentials" + + monkeypatch.setattr(identity_pool, "Credentials", credentials) config = google_credentials.load_google_credentials( prefer_workload_identity=True ) - assert config["audience"].startswith("//iam.googleapis.com/projects/123") - assert ( - "modal-example@central.iam.gserviceaccount.com" - in config["service_account_impersonation_url"] + assert config == "credentials" + assert str(calls["audience"]).startswith( + "//iam.googleapis.com/projects/123" + ) + assert "modal-example@central.iam.gserviceaccount.com" in str( + calls["service_account_impersonation_url"] ) - token_path = tmp_path / "policyengine-observability-oidc.jwt" - assert token_path.read_text() == "signed-test-token" - assert stat.S_IMODE(token_path.stat().st_mode) == 0o600 + assert "credential_source" not in calls + supplier = calls["subject_token_supplier"] + assert supplier.get_subject_token(None, None) == "signed-test-token" def test_existing_credentials_path_is_loaded(monkeypatch, tmp_path) -> None: diff --git a/tests/test_otel.py b/tests/test_otel.py index 6806219..c6ce59c 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -25,7 +25,6 @@ _google_http_session, _google_id_token_credentials, _workload_identity_audience, - _write_subject_token, ) from policyengine_observability.otel import ( OTelRuntime, @@ -306,10 +305,8 @@ def test_google_id_token_uses_modal_workload_identity(monkeypatch) -> None: calls: dict[str, object] = {} class Source: - @classmethod - def from_info(cls, config, scopes): - calls["source"] = (config, scopes) - return "source-credentials" + def __init__(self, **kwargs): + calls["source"] = kwargs class Target: def __init__(self, **kwargs): @@ -324,10 +321,6 @@ def __init__(self, **kwargs): monkeypatch.setattr( impersonated_credentials, "IDTokenCredentials", Identity ) - monkeypatch.setattr( - "policyengine_observability.google_auth._write_subject_token", - lambda token: f"/tmp/{token}", - ) monkeypatch.setenv("MODAL_IDENTITY_TOKEN", "modal-token") monkeypatch.setenv( "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER", @@ -339,9 +332,9 @@ def __init__(self, **kwargs): ) result = _google_id_token_credentials("https://collector") assert isinstance(result, Identity) - assert calls["source"][0]["credential_source"]["file"] == ( - "/tmp/modal-token" - ) + assert "credential_source" not in calls["source"] + supplier = calls["source"]["subject_token_supplier"] + assert supplier.get_subject_token(None, None) == "modal-token" assert calls["identity"]["target_audience"] == "https://collector" @@ -361,7 +354,7 @@ def test_google_id_token_falls_back_to_application_credentials( assert result[0] == "https://collector" -def test_google_transport_helpers(monkeypatch, tmp_path) -> None: +def test_google_transport_helpers(monkeypatch) -> None: import grpc from google.auth.transport import grpc as google_grpc from google.auth.transport import requests as google_requests @@ -405,9 +398,6 @@ def __init__(self, credentials): assert session.credentials == "token:https://collector" assert session.headers == {"x": "y"} - monkeypatch.setattr("tempfile.gettempdir", lambda: str(tmp_path)) - path = _write_subject_token("short-lived-token") - assert open(path).read() == "short-lived-token" assert _workload_identity_audience("projects/123/provider") == ( "//iam.googleapis.com/projects/123/provider" ) From 2a2568e9e51ca012e82532dc82b8602172dbb524 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:55:14 +0400 Subject: [PATCH 10/12] Redact attributes and isolate destination records --- README.md | 8 ++- changelog.d/29.breaking.md | 4 ++ policyengine_observability/delivery.py | 5 +- .../destinations/base.py | 3 +- .../destinations/stdout.py | 3 +- policyengine_observability/schema.py | 22 +++--- tests/test_config_schema.py | 23 +++++- tests/test_destination_strategies.py | 71 +++++++++++++++++++ tests/test_otel.py | 26 +++++++ 9 files changed, 149 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 8aea30a..5721b1c 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,13 @@ logging = LoggingConfig( The formatter adds Cloud Logging trace-correlation fields only to the output it formats. The canonical record retains the portable `trace_id`, `span_id`, -and `trace_sampled` fields. +and `trace_sampled` fields. Each destination and formatter receives a deep +copy of the canonical record, so mutations to nested values remain local to +that destination. + +Configured sensitive values are replaced in messages, exception details, and +allowlisted string attributes before length limits are applied and before the +record reaches a logging destination, span, or metric exporter. Every queued destination has its own bounded queue and worker. A blocked or failing destination cannot delay another destination or the application diff --git a/changelog.d/29.breaking.md b/changelog.d/29.breaking.md index c5a77dd..f524089 100644 --- a/changelog.d/29.breaking.md +++ b/changelog.d/29.breaking.md @@ -14,3 +14,7 @@ Inline writer cleanup now runs outside application execution and respects the configured logging shutdown timeout. Google credential JSON and workload identity subject tokens are passed to Google Auth in memory instead of being written to temporary files. +Configured sensitive values are redacted from allowlisted string attributes, +including values sent to logs, spans, and metrics. Each logging destination and +formatter now receives a deep copy so nested mutations cannot affect another +destination or the canonical record. diff --git a/policyengine_observability/delivery.py b/policyengine_observability/delivery.py index 377562d..4facaab 100644 --- a/policyengine_observability/delivery.py +++ b/policyengine_observability/delivery.py @@ -4,6 +4,7 @@ import sys import threading import time +from copy import deepcopy from dataclasses import dataclass from typing import Any, TextIO @@ -100,7 +101,7 @@ def emit(self, record: dict[str, Any]) -> None: disabled: list[_InlineDestination] = [] for destination in tuple(self._inline): try: - destination.writer.write(record.copy()) + destination.writer.write(deepcopy(record)) destination.failures = 0 except Exception as exc: destination.failures += 1 @@ -116,7 +117,7 @@ def emit(self, record: dict[str, Any]) -> None: for destination in disabled: self._disable_inline(destination) for destination in tuple(self._queued): - destination.enqueue(record.copy()) + destination.enqueue(deepcopy(record)) def _disable_inline(self, destination: _InlineDestination) -> None: try: diff --git a/policyengine_observability/destinations/base.py b/policyengine_observability/destinations/base.py index 69f2535..b8a8323 100644 --- a/policyengine_observability/destinations/base.py +++ b/policyengine_observability/destinations/base.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Callable +from copy import deepcopy from dataclasses import dataclass from typing import Any, Literal, Protocol, TextIO, runtime_checkable @@ -69,7 +70,7 @@ def close(self) -> None: close() def _format(self, record: dict[str, Any]) -> dict[str, Any]: - value = record.copy() + value = deepcopy(record) return self._formatter(value) if self._formatter else value diff --git a/policyengine_observability/destinations/stdout.py b/policyengine_observability/destinations/stdout.py index 1da438f..e219c67 100644 --- a/policyengine_observability/destinations/stdout.py +++ b/policyengine_observability/destinations/stdout.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from copy import deepcopy from dataclasses import dataclass from typing import Any, Literal @@ -21,7 +22,7 @@ def __init__( self._formatter = formatter def write(self, record: dict[str, Any]) -> None: - value = record.copy() + value = deepcopy(record) if self._formatter is not None: value = self._formatter(value) print( diff --git a/policyengine_observability/schema.py b/policyengine_observability/schema.py index 5ad9877..faec9c4 100644 --- a/policyengine_observability/schema.py +++ b/policyengine_observability/schema.py @@ -49,7 +49,7 @@ def normalize_attributes( ): omitted += 1 continue - scalar = _normalize_scalar(value, config.limits.max_string_length) + scalar = _normalize_scalar(value, config) if scalar is None: omitted += 1 continue @@ -89,7 +89,9 @@ def build_record( if event_name: record["event.name"] = _limit(event_name, config) if message: - record["message"] = _redact(_limit(message, config), config) + record["message"] = _redact(message, config)[ + : config.limits.max_string_length + ] for key, value in (context or {}).items(): if value is not None: @@ -138,17 +140,17 @@ def _error_fields( stack = "" return { "error.type": type(error).__name__, - "error.message": _redact( - message[: config.limits.max_error_message_length], config - ), - "error.stack": _redact( - stack[: config.limits.max_stack_length], config - ), + "error.message": _redact(message, config)[ + : config.limits.max_error_message_length + ], + "error.stack": _redact(stack, config)[ + : config.limits.max_stack_length + ], } def _normalize_scalar( - value: Any, max_length: int + value: Any, config: ObservabilityConfig ) -> str | int | float | bool | None: if value is None: return None @@ -161,7 +163,7 @@ def _normalize_scalar( if isinstance(value, float): return value if math.isfinite(value) else None if isinstance(value, str): - return value[:max_length] + return _redact(value, config)[: config.limits.max_string_length] return None diff --git a/tests/test_config_schema.py b/tests/test_config_schema.py index 04989f2..3bc1193 100644 --- a/tests/test_config_schema.py +++ b/tests/test_config_schema.py @@ -313,9 +313,10 @@ def test_schema_preserves_core_fields_and_namespaces_attributes() -> None: def test_schema_redacts_and_truncates_errors() -> None: config = make_config( + application_attribute_keys=frozenset({"backend"}), sensitive_values=("secret-value",), limits=TelemetryLimits( - max_string_length=8, + max_string_length=64, max_error_message_length=64, max_stack_length=100, ), @@ -327,13 +328,33 @@ def test_schema_redacts_and_truncates_errors() -> None: config, severity="ERROR", message="secret-value message", + attributes={"backend": "prefix-secret-value-suffix"}, error=error, ) assert "secret-value" not in str(record) assert "[REDACTED]" in str(record) + assert record["attributes"]["backend"] == "prefix-[REDACTED]-suffix" assert len(record["error.stack"]) <= 100 +def test_schema_redacts_before_truncating_strings() -> None: + config = make_config( + application_attribute_keys=frozenset({"backend"}), + sensitive_values=("secret-value",), + limits=TelemetryLimits(max_string_length=8), + ) + + record = build_record( + config, + severity="INFO", + message="secret-value", + attributes={"backend": "secret-value"}, + ) + + assert record["message"] == "[REDACTE" + assert record["attributes"]["backend"] == "[REDACTE" + + def test_attribute_policy_omits_sensitive_non_scalar_and_nonfinite() -> None: config = make_config( application_attribute_keys=frozenset( diff --git a/tests/test_destination_strategies.py b/tests/test_destination_strategies.py index b9df376..5371b24 100644 --- a/tests/test_destination_strategies.py +++ b/tests/test_destination_strategies.py @@ -14,6 +14,7 @@ StdoutLogDestination, ) from policyengine_observability.delivery import DeliveryManager +from policyengine_observability.destinations import DestinationBuildContext from policyengine_observability.diagnostics import Diagnostics @@ -162,6 +163,76 @@ def test_one_destination_failure_does_not_affect_another_destination() -> None: assert diagnostics.count("failure.logging.destination_disabled") == 1 +def test_formatters_receive_deeply_isolated_records() -> None: + formatted = RecordingWriter() + unchanged = RecordingWriter() + source = {"attributes": {"backend": "original"}} + + def mutate_nested(record: dict) -> dict: + record["attributes"]["backend"] = "formatted" + return record + + manager = DeliveryManager( + make_config( + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="formatted", + writer_factory=lambda: formatted, + delivery="inline", + formatter=mutate_nested, + ), + CustomLogDestination( + name="unchanged", + writer_factory=lambda: unchanged, + delivery="inline", + ), + ) + ) + ), + Diagnostics(), + ) + + manager.emit(source) + + assert formatted.records[0]["attributes"]["backend"] == "formatted" + assert unchanged.records[0]["attributes"]["backend"] == "original" + assert source["attributes"]["backend"] == "original" + manager.close(1) + + +def test_destination_writers_copy_nested_values_before_formatting() -> None: + source = {"attributes": {"backend": "original"}} + + def mutate_nested(record: dict) -> dict: + record["attributes"]["backend"] = "formatted" + return record + + output = io.StringIO() + context = DestinationBuildContext(stdout=lambda: output) + stdout_writer = StdoutLogDestination(formatter=mutate_nested).build_writer( + context + ) + stdout_writer.write(source) + + assert json.loads(output.getvalue())["attributes"]["backend"] == ( + "formatted" + ) + assert source["attributes"]["backend"] == "original" + + recording = RecordingWriter() + custom_writer = CustomLogDestination( + name="custom", + writer_factory=lambda: recording, + delivery="inline", + formatter=mutate_nested, + ).build_writer(context) + custom_writer.write(source) + + assert recording.records[0]["attributes"]["backend"] == "formatted" + assert source["attributes"]["backend"] == "original" + + def test_slow_remote_destination_does_not_delay_another_destination() -> None: started = threading.Event() release = threading.Event() diff --git a/tests/test_otel.py b/tests/test_otel.py index c6ce59c..bd0fea7 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -72,6 +72,32 @@ def test_owned_provider_creates_local_spans_metrics_and_resources() -> None: runtime.shutdown() +def test_sensitive_values_are_redacted_from_span_attributes() -> None: + config = make_config( + otel=OTelConfig(enabled=True), + application_attribute_keys=frozenset({"backend"}), + sensitive_values=("secret-value",), + ) + runtime = configure(config) + runtime._delivery._stdout = io.StringIO() + exporter = InMemorySpanExporter() + runtime._otel._tracer_provider.add_span_processor( + SimpleSpanProcessor(exporter) + ) + + with runtime.operation( + "simulation.run", + attributes={"backend": "prefix-secret-value-suffix"}, + ): + pass + + span = exporter.get_finished_spans()[0] + assert span.attributes["backend"] == "prefix-[REDACTED]-suffix" + item = records(runtime._delivery._stdout)[0] + assert item["attributes"]["backend"] == "prefix-[REDACTED]-suffix" + runtime.shutdown() + + def test_incoming_trace_context_correlates_log_and_response() -> None: runtime, exporter = _runtime_with_spans() trace_id = "1" * 32 From 3fccd93d4b87681f12b92c4f6f77ef7f2f9e6e1f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 22 Sep 2026 23:15:07 +0400 Subject: [PATCH 11/12] Validate redaction config and contain Flask setup failures --- README.md | 5 +- changelog.d/29.breaking.md | 3 + policyengine_observability/adapters/flask.py | 71 ++++++++++++++++-- policyengine_observability/config.py | 12 +++ tests/test_adapters.py | 78 ++++++++++++++++++++ tests/test_config_schema.py | 20 +++++ 6 files changed, 182 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 5721b1c..951a422 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,10 @@ instrument_fastapi(app, runtime) Both adapters extract W3C trace context and `X-PolicyEngine-Request-Id`, create one server span, emit one completion record, record request metrics, and clear context-local state. Repeated calls -reuse the first runtime associated with the application. +reuse the first runtime associated with the application. Install the Flask +adapter before serving the first request. If Flask rejects callback +registration, the adapter restores the prior callback registries, leaves the +application unmarked, reports a local diagnostic, and returns without raising. ## Application instrumentation diff --git a/changelog.d/29.breaking.md b/changelog.d/29.breaking.md index f524089..db8c8da 100644 --- a/changelog.d/29.breaking.md +++ b/changelog.d/29.breaking.md @@ -18,3 +18,6 @@ Configured sensitive values are redacted from allowlisted string attributes, including values sent to logs, spans, and metrics. Each logging destination and formatter now receives a deep copy so nested mutations cannot affect another destination or the canonical record. +Invalid `sensitive_values` entries now fail configuration validation before +runtime setup. Flask callback installation failures restore the prior +application state and do not propagate into application execution. diff --git a/policyengine_observability/adapters/flask.py b/policyengine_observability/adapters/flask.py index 7801da4..1872fa1 100644 --- a/policyengine_observability/adapters/flask.py +++ b/policyengine_observability/adapters/flask.py @@ -5,6 +5,12 @@ from ..runtime import ObservabilityRuntime _EXTENSION_KEY = "policyengine_observability" +_CALLBACK_REGISTRIES = ( + "before_request_funcs", + "after_request_funcs", + "teardown_request_funcs", +) +_MISSING = object() def instrument_flask( @@ -12,13 +18,15 @@ def instrument_flask( ) -> ObservabilityRuntime: """Install one request lifecycle integration on a Flask application.""" - existing = app.extensions.get(_EXTENSION_KEY) + try: + extensions = app.extensions + existing = extensions.get(_EXTENSION_KEY, _MISSING) + except Exception as exc: + runtime.diagnostics.report("flask.callback_install", exc) + return runtime if isinstance(existing, ObservabilityRuntime): return existing - app.extensions[_EXTENSION_KEY] = runtime - - @app.before_request def _begin_observed_request() -> None: try: from flask import request @@ -34,7 +42,6 @@ def _begin_observed_request() -> None: except Exception as exc: runtime.diagnostics.report("flask.request_begin", exc) - @app.after_request def _finish_observed_request(response: Any) -> Any: try: from flask import request @@ -48,7 +55,6 @@ def _finish_observed_request(response: Any) -> Any: runtime.diagnostics.report("flask.request_finish", exc) return response - @app.teardown_request def _close_observed_request(error: BaseException | None) -> None: try: runtime.end_request( @@ -58,4 +64,57 @@ def _close_observed_request(error: BaseException | None) -> None: except Exception as exc: runtime.diagnostics.report("flask.request_teardown", exc) + snapshots: dict[str, dict[Any, list[Any]]] | None = None + try: + snapshots = _snapshot_callback_registries(app) + app.before_request(_begin_observed_request) + app.after_request(_finish_observed_request) + app.teardown_request(_close_observed_request) + extensions[_EXTENSION_KEY] = runtime + except Exception as exc: + if snapshots is not None: + _restore_callback_registries(app, snapshots, runtime) + try: + if existing is _MISSING: + extensions.pop(_EXTENSION_KEY, None) + else: + extensions[_EXTENSION_KEY] = existing + except Exception as rollback_error: + runtime.diagnostics.report( + "flask.callback_install_rollback", rollback_error + ) + runtime.diagnostics.report("flask.callback_install", exc) + return runtime + + +def _snapshot_callback_registries( + app: Any, +) -> dict[str, dict[Any, list[Any]]]: + return { + name: { + key: list(callbacks) + for key, callbacks in getattr(app, name).items() + } + for name in _CALLBACK_REGISTRIES + } + + +def _restore_callback_registries( + app: Any, + snapshots: dict[str, dict[Any, list[Any]]], + runtime: ObservabilityRuntime, +) -> None: + for name, snapshot in snapshots.items(): + try: + registry = getattr(app, name) + registry.clear() + registry.update( + {key: list(callbacks) for key, callbacks in snapshot.items()} + ) + except Exception as exc: + runtime.diagnostics.report( + "flask.callback_install_rollback", + exc, + registry=name, + ) diff --git a/policyengine_observability/config.py b/policyengine_observability/config.py index 104abb2..ad70d96 100644 --- a/policyengine_observability/config.py +++ b/policyengine_observability/config.py @@ -264,6 +264,18 @@ def validation_errors(self) -> tuple[str, ...]: if not isinstance(value, str) or not value.strip(): errors.append(f"{key} must be a non-empty string.") + if not isinstance(self.sensitive_values, tuple): + errors.append( + "sensitive_values must be a tuple of non-empty strings." + ) + else: + for index, value in enumerate(self.sensitive_values): + if not isinstance(value, str) or not value.strip(): + errors.append( + f"sensitive_values[{index}] must be a non-empty " + "string." + ) + _choice_error( errors, "deployment.platform", diff --git a/tests/test_adapters.py b/tests/test_adapters.py index 9b020ca..51e4a1d 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -1,8 +1,10 @@ from __future__ import annotations import asyncio +from typing import Any import httpx +import pytest from conftest import make_runtime, records from fastapi import FastAPI from fastapi.testclient import TestClient @@ -56,6 +58,66 @@ def fail(): runtime.shutdown() +def test_flask_late_installation_is_nonfatal_and_leaves_no_state() -> None: + runtime, output = make_runtime() + app = Flask(__name__) + + @app.get("/health") + def health(): + return {"status": "ok"} + + client = app.test_client() + assert client.get("/health").status_code == 200 + callbacks_before = _flask_callback_snapshot(app) + + assert instrument_flask(app, runtime) is runtime + + assert _flask_callback_snapshot(app) == callbacks_before + assert "policyengine_observability" not in app.extensions + assert runtime.diagnostics.count("failure.flask.callback_install") == 1 + assert client.get("/health").status_code == 200 + assert records(output) == [] + runtime.shutdown() + + +@pytest.mark.parametrize( + "registration_name", + ("before_request", "after_request", "teardown_request"), +) +def test_flask_installation_failure_rolls_back_and_can_retry( + monkeypatch, registration_name +) -> None: + runtime, output = make_runtime() + app = Flask(__name__) + + @app.get("/health") + def health(): + return {"status": "ok"} + + callbacks_before = _flask_callback_snapshot(app) + original = getattr(app, registration_name) + + def register_then_fail(callback): + original(callback) + raise RuntimeError("registration failed") + + monkeypatch.setattr(app, registration_name, register_then_fail) + + assert instrument_flask(app, runtime) is runtime + assert _flask_callback_snapshot(app) == callbacks_before + assert "policyengine_observability" not in app.extensions + assert runtime.diagnostics.count("failure.flask.callback_install") == 1 + + monkeypatch.setattr(app, registration_name, original) + assert instrument_flask(app, runtime) is runtime + response = app.test_client().get("/health") + + assert response.status_code == 200 + assert REQUEST_ID_HEADER in response.headers + assert len(records(output)) == 1 + runtime.shutdown() + + def test_fastapi_lifecycle_and_idempotence() -> None: runtime, output = make_runtime() unused, _ = make_runtime() @@ -192,3 +254,19 @@ def test_httpx_client_cannot_be_rebound_to_another_runtime() -> None: client.close() first.shutdown() second.shutdown() + + +def _flask_callback_snapshot( + app: Flask, +) -> dict[str, dict[Any, tuple[Any, ...]]]: + return { + name: { + key: tuple(callbacks) + for key, callbacks in getattr(app, name).items() + } + for name in ( + "before_request_funcs", + "after_request_funcs", + "teardown_request_funcs", + ) + } diff --git a/tests/test_config_schema.py b/tests/test_config_schema.py index 3bc1193..7b0b975 100644 --- a/tests/test_config_schema.py +++ b/tests/test_config_schema.py @@ -124,6 +124,26 @@ def test_invalid_nested_limits_are_reported_together() -> None: assert field in message +@pytest.mark.parametrize( + ("sensitive_values", "expected"), + [ + (["secret"], "sensitive_values must be a tuple"), + ((123,), r"sensitive_values\[0\] must be a non-empty string"), + (("",), r"sensitive_values\[0\] must be a non-empty string"), + ((" ",), r"sensitive_values\[0\] must be a non-empty string"), + ], +) +def test_invalid_sensitive_values_fail_before_runtime_setup( + sensitive_values, expected +) -> None: + config = make_config( + sensitive_values=sensitive_values, # type: ignore[arg-type] + ) + + with pytest.raises(ConfigurationError, match=expected): + configure(config) + + def test_invalid_destination_strategies_are_reported() -> None: class InvalidDestination: name = "invalid" From 8ef4b4749534ed6d0f54288fd3a598d65137b955 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 22 Sep 2026 23:37:30 +0400 Subject: [PATCH 12/12] Fix telemetry redaction and Flask lifecycle --- README.md | 12 +++- changelog.d/29.breaking.md | 6 ++ policyengine_observability/adapters/flask.py | 31 ++++++++++- policyengine_observability/config.py | 28 +++++++++- policyengine_observability/diagnostics.py | 31 ++++++++--- policyengine_observability/otel.py | 29 ++++++++-- policyengine_observability/runtime.py | 33 ++++++++--- policyengine_observability/schema.py | 4 +- tests/test_adapters.py | 38 ++++++++++++- tests/test_config_schema.py | 46 ++++++++++++++++ tests/test_diagnostics_public_api.py | 20 +++++++ tests/test_otel.py | 58 +++++++++++++++++--- tests/test_otel_destinations.py | 2 + 13 files changed, 298 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 951a422..4aa7d87 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,8 @@ that destination. Configured sensitive values are replaced in messages, exception details, and allowlisted string attributes before length limits are applied and before the -record reaches a logging destination, span, or metric exporter. +record reaches a logging destination, span, or metric exporter. This also +applies to exception events on spans and local internal diagnostics. Every queued destination has its own bounded queue and worker. A blocked or failing destination cannot delay another destination or the application @@ -162,6 +163,11 @@ otel = OTelConfig( ) ``` +For OTLP over HTTP, the default `endpoint_mode="base"` appends +`/v1/traces` or `/v1/metrics` to the configured endpoint. Set +`endpoint_mode="signal"` when the endpoint already identifies the exact +signal route. + For a Google ID-token protected collector, select the authentication strategy explicitly: @@ -186,6 +192,10 @@ OTEL_EXPORTER_OTLP_TRACES_HEADERS=x-api-key=trace-key OTEL_EXPORTER_OTLP_METRICS_HEADERS=x-api-key=metric-key ``` +The common HTTP endpoint is treated as a base URL. Signal-specific HTTP +endpoint variables are passed to the exporter exactly as configured, following +the OpenTelemetry environment-variable contract. + `POLICYENGINE_OTEL_GOOGLE_AUDIENCE` selects `GoogleIdTokenAuth` for both signals. `POLICYENGINE_OTEL_TRACES_GOOGLE_AUDIENCE` and `POLICYENGINE_OTEL_METRICS_GOOGLE_AUDIENCE` override it per signal. diff --git a/changelog.d/29.breaking.md b/changelog.d/29.breaking.md index db8c8da..4864bc3 100644 --- a/changelog.d/29.breaking.md +++ b/changelog.d/29.breaking.md @@ -21,3 +21,9 @@ destination or the canonical record. Invalid `sensitive_values` entries now fail configuration validation before runtime setup. Flask callback installation failures restore the prior application state and do not propagate into application execution. +Configured sensitive values are also redacted from OpenTelemetry exception +events and local diagnostics. Attribute allowlists now require frozensets of +non-empty strings. Flask instrumentation starts before existing request +callbacks and completes during request teardown so failures retain exception +details. HTTP 5xx responses set span error status. Signal-specific OTLP/HTTP +environment endpoints are passed through unchanged. diff --git a/policyengine_observability/adapters/flask.py b/policyengine_observability/adapters/flask.py index 1872fa1..1bb0ef5 100644 --- a/policyengine_observability/adapters/flask.py +++ b/policyengine_observability/adapters/flask.py @@ -48,11 +48,17 @@ def _finish_observed_request(response: Any) -> Any: if request.url_rule is not None: runtime.update_request_route(request.url_rule.rule) + except Exception as exc: + runtime.diagnostics.report("flask.request_route", exc) + try: + runtime.update_request_status(response.status_code) + except Exception as exc: + runtime.diagnostics.report("flask.request_status", exc) + try: for key, value in runtime.response_headers().items(): response.headers[key] = value - runtime.end_request(status_code=response.status_code) except Exception as exc: - runtime.diagnostics.report("flask.request_finish", exc) + runtime.diagnostics.report("flask.response_headers", exc) return response def _close_observed_request(error: BaseException | None) -> None: @@ -68,7 +74,15 @@ def _close_observed_request(error: BaseException | None) -> None: try: snapshots = _snapshot_callback_registries(app) app.before_request(_begin_observed_request) + _move_callback_to_start( + app.before_request_funcs, + _begin_observed_request, + ) app.after_request(_finish_observed_request) + _move_callback_to_start( + app.after_request_funcs, + _finish_observed_request, + ) app.teardown_request(_close_observed_request) extensions[_EXTENSION_KEY] = runtime except Exception as exc: @@ -88,6 +102,19 @@ def _close_observed_request(error: BaseException | None) -> None: return runtime +def _move_callback_to_start( + registry: dict[Any, list[Any]], callback: Any +) -> None: + callbacks = registry.get(None) + if callbacks is None: + raise RuntimeError("Flask did not register the callback.") + for index, registered in enumerate(callbacks): + if registered is callback: + callbacks.insert(0, callbacks.pop(index)) + return + raise RuntimeError("Flask did not register the callback.") + + def _snapshot_callback_registries( app: Any, ) -> dict[str, dict[Any, list[Any]]]: diff --git a/policyengine_observability/config.py b/policyengine_observability/config.py index ad70d96..6bada68 100644 --- a/policyengine_observability/config.py +++ b/policyengine_observability/config.py @@ -9,6 +9,7 @@ Platform = Literal["google_cloud_run", "modal", "local", "other"] OTLPProtocol = Literal["grpc", "http/protobuf"] +OTLPEndpointMode = Literal["base", "signal"] ProviderMode = Literal["owned", "external"] @@ -83,6 +84,7 @@ def exporter_kwargs( class OTLPExporterConfig: endpoint: str protocol: OTLPProtocol = "grpc" + endpoint_mode: OTLPEndpointMode = "base" headers: tuple[tuple[str, str], ...] = () auth: OTLPAuthentication | None = None timeout_seconds: float = 5.0 @@ -276,6 +278,20 @@ def validation_errors(self) -> tuple[str, ...]: "string." ) + for name, values in ( + ("application_attribute_keys", self.application_attribute_keys), + ("dispatch_attribute_keys", self.dispatch_attribute_keys), + ("metric_attribute_keys", self.metric_attribute_keys), + ): + if not isinstance(values, frozenset): + errors.append( + f"{name} must be a frozenset of non-empty strings." + ) + continue + for value in values: + if not isinstance(value, str) or not value.strip(): + errors.append(f"{name} entries must be non-empty strings.") + _choice_error( errors, "deployment.platform", @@ -390,6 +406,12 @@ def validation_errors(self) -> tuple[str, ...]: exporter.protocol, {"grpc", "http/protobuf"}, ) + _choice_error( + errors, + f"otel.{signal}.endpoint_mode", + exporter.endpoint_mode, + {"base", "signal"}, + ) _number_error( errors, f"otel.{signal}.timeout_seconds", @@ -492,7 +514,8 @@ def _exporter_from_env( if not enabled: return None prefix = f"OTEL_EXPORTER_OTLP_{signal.upper()}" - endpoint = os.getenv(f"{prefix}_ENDPOINT") or common_endpoint + signal_endpoint = os.getenv(f"{prefix}_ENDPOINT") + endpoint = signal_endpoint or common_endpoint if not endpoint: return None protocol_value = os.getenv(f"{prefix}_PROTOCOL") @@ -533,6 +556,7 @@ def _exporter_from_env( return OTLPExporterConfig( endpoint=endpoint, protocol=protocol, + endpoint_mode="signal" if signal_endpoint else "base", headers=headers, auth=auth, timeout_seconds=timeout / 1_000, @@ -602,7 +626,7 @@ def _env_int( def _choice_error( errors: list[str], name: str, value: Any, choices: set[str] ) -> None: - if value not in choices: + if not isinstance(value, str) or value not in choices: errors.append(f"{name} must be one of: {', '.join(sorted(choices))}.") diff --git a/policyengine_observability/diagnostics.py b/policyengine_observability/diagnostics.py index 838723d..79050d8 100644 --- a/policyengine_observability/diagnostics.py +++ b/policyengine_observability/diagnostics.py @@ -18,9 +18,11 @@ def __init__( *, interval_seconds: float = 60.0, stderr: Any = None, + sensitive_values: tuple[str, ...] = (), ) -> None: self._interval_seconds = max(1.0, interval_seconds) self._stderr = stderr or sys.stderr + self._sensitive_values = sensitive_values self._last_report: dict[str, float] = {} self._counts: Counter[str] = Counter() self._listeners: list[Callable[[str, int], None]] = [] @@ -79,8 +81,11 @@ def report( "event.name": "observability.internal_failure", "operation": operation, "error.type": type(error).__name__, - "error.message": _safe_text(error), - **{str(key): _safe_scalar(value) for key, value in fields.items()}, + "error.message": _safe_text(error, self._sensitive_values), + **{ + str(key): _safe_scalar(value, self._sensitive_values) + for key, value in fields.items() + }, } try: print( @@ -92,14 +97,22 @@ def report( return -def _safe_text(value: Any) -> str: +def _safe_text(value: Any, sensitive_values: tuple[str, ...]) -> str: try: - return str(value)[:2_048] + text = str(value) except Exception: return "" - - -def _safe_scalar(value: Any) -> str | int | float | bool | None: - if value is None or isinstance(value, (str, int, float, bool)): + for sensitive in sensitive_values: + if sensitive: + text = text.replace(sensitive, "[REDACTED]") + return text[:2_048] + + +def _safe_scalar( + value: Any, sensitive_values: tuple[str, ...] +) -> str | int | float | bool | None: + if isinstance(value, str): + return _safe_text(value, sensitive_values) + if value is None or isinstance(value, (int, float, bool)): return value - return _safe_text(value) + return _safe_text(value, sensitive_values) diff --git a/policyengine_observability/otel.py b/policyengine_observability/otel.py index 5596b6f..4cd91b2 100644 --- a/policyengine_observability/otel.py +++ b/policyengine_observability/otel.py @@ -8,7 +8,7 @@ from .config import ObservabilityConfig, OTLPExporterConfig from .diagnostics import Diagnostics -from .schema import metric_attributes +from .schema import error_fields, metric_attributes @dataclass(slots=True) @@ -237,15 +237,26 @@ def end_span( self, handle: SpanHandle | None, error: BaseException | None = None, + *, + failed: bool = False, ) -> None: if handle is None: return try: - if error is not None: + if error is not None or failed: from opentelemetry.trace import Status, StatusCode - if isinstance(error, Exception): - handle.span.record_exception(error) + if error is not None: + details = error_fields(error, self.config) + handle.span.add_event( + "exception", + attributes={ + "exception.type": details["error.type"], + "exception.message": details["error.message"], + "exception.stacktrace": details["error.stack"], + "exception.escaped": False, + }, + ) handle.span.set_status(Status(StatusCode.ERROR)) handle.manager.__exit__( type(error) if error is not None else None, @@ -546,7 +557,10 @@ def _build_span_exporter(config: OTLPExporterConfig) -> Any: OTLPSpanExporter, ) - kwargs["endpoint"] = _http_signal_endpoint(config.endpoint, "traces") + if config.endpoint_mode == "base": + kwargs["endpoint"] = _http_signal_endpoint( + config.endpoint, "traces" + ) return OTLPSpanExporter(**kwargs) from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( OTLPSpanExporter, @@ -562,7 +576,10 @@ def _build_metric_exporter(config: OTLPExporterConfig) -> Any: OTLPMetricExporter, ) - kwargs["endpoint"] = _http_signal_endpoint(config.endpoint, "metrics") + if config.endpoint_mode == "base": + kwargs["endpoint"] = _http_signal_endpoint( + config.endpoint, "metrics" + ) return OTLPMetricExporter(**kwargs) from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( OTLPMetricExporter, diff --git a/policyengine_observability/runtime.py b/policyengine_observability/runtime.py index 9af7fda..1b1d55c 100644 --- a/policyengine_observability/runtime.py +++ b/policyengine_observability/runtime.py @@ -66,7 +66,9 @@ class ObservabilityRuntime: def __init__(self, config: ObservabilityConfig) -> None: config.validate() self.config = config - self.diagnostics = Diagnostics() + self.diagnostics = Diagnostics( + sensitive_values=config.sensitive_values, + ) self._request_state: ContextVar[_RequestState | None] = ContextVar( f"policyengine_request_{id(self)}", default=None ) @@ -231,6 +233,12 @@ def update_request_route(self, route: str) -> None: state.route = route self._otel.set_span_attributes({"http.route": route}) + def update_request_status(self, status_code: int) -> None: + state = self._request_state.get() + if state is None or state.completed: + return + state.status_code = status_code + def response_headers(self) -> dict[str, str]: state = self._request_state.get() if state is None: @@ -252,16 +260,21 @@ def end_request( if state is None or state.completed: return state.completed = True - state.status_code = status_code + resolved_status = ( + status_code if status_code is not None else state.status_code + ) + state.status_code = resolved_status state.error = error duration = max(0.0, time.perf_counter() - state.start_time) - outcome = _outcome(status_code, error) + outcome = _outcome(resolved_status, error) metric_values = self._metric_base() metric_values.update( { "http.route": state.route, "http.request.method": state.method, - "http.response.status_code_class": _status_class(status_code), + "http.response.status_code_class": _status_class( + resolved_status + ), "operation.kind": "request", "outcome": outcome, } @@ -269,7 +282,7 @@ def end_request( self._otel.set_span_attributes( { "http.route": state.route, - "http.response.status_code": status_code or 0, + "http.response.status_code": resolved_status or 0, "policyengine.request.id": state.request_id, "policyengine.outcome": outcome, } @@ -278,7 +291,7 @@ def end_request( "request.id": state.request_id, "http.request.method": state.method, "http.route": state.route, - "http.response.status_code": status_code, + "http.response.status_code": resolved_status, "duration_ms": round(duration * 1_000, 3), "outcome": outcome, **self._otel.current_correlation(), @@ -291,9 +304,13 @@ def end_request( error=error, ) self._otel.record_request(duration, metric_values) - if error is not None: + if outcome == "error": self._otel.record_error(metric_values) - self._otel.end_span(state.span, error) + self._otel.end_span( + state.span, + error, + failed=outcome == "error", + ) self._reset_request(state) def set_context(self, **attributes: Any) -> None: diff --git a/policyengine_observability/schema.py b/policyengine_observability/schema.py index faec9c4..7133b1c 100644 --- a/policyengine_observability/schema.py +++ b/policyengine_observability/schema.py @@ -109,7 +109,7 @@ def build_record( record["attributes.omitted_count"] = omitted if error is not None: - record.update(_error_fields(error, config)) + record.update(error_fields(error, config)) return record @@ -125,7 +125,7 @@ def metric_attributes( return normalized -def _error_fields( +def error_fields( error: BaseException, config: ObservabilityConfig ) -> dict[str, Any]: try: diff --git a/tests/test_adapters.py b/tests/test_adapters.py index 51e4a1d..a157728 100644 --- a/tests/test_adapters.py +++ b/tests/test_adapters.py @@ -9,9 +9,15 @@ from fastapi import FastAPI from fastapi.testclient import TestClient from flask import Flask +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.trace import StatusCode from policyengine_observability import ( REQUEST_ID_HEADER, + OTelConfig, instrument_fastapi, instrument_flask, instrument_httpx, @@ -42,7 +48,11 @@ def item(item_id: str): def test_flask_exception_emits_one_error_completion() -> None: - runtime, output = make_runtime() + runtime, output = make_runtime(otel=OTelConfig(enabled=True)) + exporter = InMemorySpanExporter() + runtime._otel._tracer_provider.add_span_processor( + SimpleSpanProcessor(exporter) + ) app = Flask(__name__) @app.get("/fail") @@ -55,6 +65,32 @@ def fail(): emitted = records(output) assert len(emitted) == 1 assert emitted[0]["outcome"] == "error" + assert emitted[0]["error.type"] == "ValueError" + assert emitted[0]["error.message"] == "route failed" + span = exporter.get_finished_spans()[0] + assert span.status.status_code is StatusCode.ERROR + assert span.events[0].name == "exception" + assert span.events[0].attributes["exception.message"] == "route failed" + runtime.shutdown() + + +def test_flask_observes_early_before_request_response() -> None: + runtime, output = make_runtime() + app = Flask(__name__) + + @app.before_request + def reject_request(): + return {"error": "unauthorized"}, 401 + + instrument_flask(app, runtime) + response = app.test_client().get("/protected") + + assert response.status_code == 401 + assert REQUEST_ID_HEADER in response.headers + emitted = records(output) + assert len(emitted) == 1 + assert emitted[0]["http.response.status_code"] == 401 + assert emitted[0]["outcome"] == "client_error" runtime.shutdown() diff --git a/tests/test_config_schema.py b/tests/test_config_schema.py index 7b0b975..5a5c699 100644 --- a/tests/test_config_schema.py +++ b/tests/test_config_schema.py @@ -81,6 +81,7 @@ def test_invalid_nested_limits_are_reported_together() -> None: exporter = OTLPExporterConfig( endpoint="", protocol="invalid", # type: ignore[arg-type] + endpoint_mode="invalid", # type: ignore[arg-type] timeout_seconds=0, ) config = make_config( @@ -117,6 +118,7 @@ def test_invalid_nested_limits_are_reported_together() -> None: "otel.shutdown_timeout_seconds", "otel.traces.endpoint", "otel.traces.protocol", + "otel.traces.endpoint_mode", "otel.traces.timeout_seconds", "limits.max_attributes", "limits.async_parent_max_age_seconds", @@ -144,6 +146,26 @@ def test_invalid_sensitive_values_fail_before_runtime_setup( configure(config) +@pytest.mark.parametrize( + ("field_name", "value"), + [ + ("application_attribute_keys", ("backend",)), + ("application_attribute_keys", frozenset({""})), + ("dispatch_attribute_keys", ["job_id"]), + ("dispatch_attribute_keys", frozenset({1})), + ("metric_attribute_keys", {"outcome"}), + ("metric_attribute_keys", frozenset({" "})), + ], +) +def test_invalid_attribute_allowlists_fail_before_runtime_setup( + field_name, value +) -> None: + config = make_config(**{field_name: value}) + + with pytest.raises(ConfigurationError, match=field_name): + configure(config) + + def test_invalid_destination_strategies_are_reported() -> None: class InvalidDestination: name = "invalid" @@ -237,6 +259,30 @@ def test_from_env_reads_transport_but_not_identity(monkeypatch) -> None: assert config.otel.metric_export_interval_seconds == 4.0 +def test_from_env_marks_signal_specific_endpoints_as_exact( + monkeypatch, +) -> None: + monkeypatch.setenv( + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "https://collector/custom-traces", + ) + monkeypatch.setenv( + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + "https://collector/custom-metrics", + ) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf") + + config = ObservabilityConfig.from_env( + service=ServiceIdentity("svc", "ns", "1", "api"), + deployment=DeploymentIdentity("dev", "local"), + ) + + assert config.otel.traces is not None + assert config.otel.metrics is not None + assert config.otel.traces.endpoint_mode == "signal" + assert config.otel.metrics.endpoint_mode == "signal" + + @pytest.mark.parametrize( ("name", "value"), [ diff --git a/tests/test_diagnostics_public_api.py b/tests/test_diagnostics_public_api.py index b8c2995..f7412b0 100644 --- a/tests/test_diagnostics_public_api.py +++ b/tests/test_diagnostics_public_api.py @@ -69,6 +69,26 @@ def test_diagnostics_are_local_rate_limited_and_counted() -> None: assert diagnostics.count("dropped") == 3 +def test_diagnostics_redact_configured_sensitive_values() -> None: + output = io.StringIO() + diagnostics = Diagnostics( + stderr=output, + sensitive_values=("secret-value",), + ) + + diagnostics.report( + "export.failed", + ValueError("secret-value exporter failure"), + destination="prefix-secret-value-suffix", + ) + + rendered = output.getvalue() + item = json.loads(rendered) + assert "secret-value" not in rendered + assert item["error.message"] == "[REDACTED] exporter failure" + assert item["destination"] == "prefix-[REDACTED]-suffix" + + def test_diagnostic_listener_failure_does_not_escape() -> None: diagnostics = Diagnostics(stderr=io.StringIO()) diagnostics.add_listener( diff --git a/tests/test_otel.py b/tests/test_otel.py index bd0fea7..3c249b1 100644 --- a/tests/test_otel.py +++ b/tests/test_otel.py @@ -72,7 +72,7 @@ def test_owned_provider_creates_local_spans_metrics_and_resources() -> None: runtime.shutdown() -def test_sensitive_values_are_redacted_from_span_attributes() -> None: +def test_sensitive_values_are_redacted_from_spans_and_logs() -> None: config = make_config( otel=OTelConfig(enabled=True), application_attribute_keys=frozenset({"backend"}), @@ -85,16 +85,27 @@ def test_sensitive_values_are_redacted_from_span_attributes() -> None: SimpleSpanProcessor(exporter) ) - with runtime.operation( - "simulation.run", - attributes={"backend": "prefix-secret-value-suffix"}, - ): + try: + with runtime.operation( + "simulation.run", + attributes={"backend": "prefix-secret-value-suffix"}, + ): + raise ValueError("secret-value operation failed") + except ValueError: pass span = exporter.get_finished_spans()[0] assert span.attributes["backend"] == "prefix-[REDACTED]-suffix" + event = span.events[0] + assert event.name == "exception" + assert event.attributes["exception.message"] == ( + "[REDACTED] operation failed" + ) + assert "secret-value" not in event.attributes["exception.stacktrace"] + assert "[REDACTED]" in event.attributes["exception.stacktrace"] item = records(runtime._delivery._stdout)[0] assert item["attributes"]["backend"] == "prefix-[REDACTED]-suffix" + assert item["error.message"] == "[REDACTED] operation failed" runtime.shutdown() @@ -132,6 +143,24 @@ def test_malformed_trace_context_starts_new_trace() -> None: runtime.shutdown() +def test_server_error_response_sets_span_error_status() -> None: + from opentelemetry.trace import StatusCode + + runtime, exporter = _runtime_with_spans() + runtime.begin_request( + headers={}, + method="GET", + route="/unavailable", + ) + + runtime.end_request(status_code=503) + + span = exporter.get_finished_spans()[0] + assert span.status.status_code is StatusCode.ERROR + assert span.events == () + runtime.shutdown() + + def test_recent_async_context_is_parent_and_old_context_is_link() -> None: runtime, exporter = _runtime_with_spans() trace_id = "3" * 32 @@ -294,6 +323,16 @@ def constructor(name): assert created[0][1]["headers"] == {"x-test": "value"} assert created[0][1]["timeout"] == 2 + exact = OTLPExporterConfig( + endpoint="https://collector/custom-signal", + protocol="http/protobuf", + endpoint_mode="signal", + ) + assert _build_span_exporter(exact) == "http-span" + assert _build_metric_exporter(exact) == "http-metric" + assert created[4][1]["endpoint"] == "https://collector/custom-signal" + assert created[5][1]["endpoint"] == "https://collector/custom-signal" + def test_google_exporter_kwargs_select_protocol_credentials( monkeypatch, @@ -487,11 +526,11 @@ def test_otel_end_span_metrics_flush_and_shutdown_fail_open() -> None: class Span: def __init__(self): - self.recorded = None + self.events = [] self.status = None - def record_exception(self, error): - self.recorded = error + def add_event(self, name, attributes): + self.events.append((name, attributes)) def set_status(self, status): self.status = status @@ -507,7 +546,8 @@ def __exit__(self, *args): manager = Manager() error = ValueError("application") otel.end_span(SpanHandle(manager=manager, span=span), error) - assert span.recorded is error + assert span.events[0][0] == "exception" + assert span.events[0][1]["exception.message"] == "application" assert manager.args[1] is error otel.end_span(None) diff --git a/tests/test_otel_destinations.py b/tests/test_otel_destinations.py index 804a06a..08e29f3 100644 --- a/tests/test_otel_destinations.py +++ b/tests/test_otel_destinations.py @@ -29,10 +29,12 @@ def test_signal_specific_otlp_environment_configuration(monkeypatch) -> None: assert config.otel.traces == OTLPExporterConfig( endpoint="https://traces", + endpoint_mode="signal", headers=(("trace-key", "trace-value"),), ) assert config.otel.metrics == OTLPExporterConfig( endpoint="https://metrics", + endpoint_mode="signal", headers=(("metric-key", "metric-value"),), )