diff --git a/.gitignore b/.gitignore index acb6a3f..234d2c8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ coverage.xml __pycache__/ *.py[cod] .DS_Store +/deploy/gcp/rendered/ diff --git a/README.md b/README.md index 3db5dde..4aa7d87 100644 --- a/README.md +++ b/README.md @@ -1,188 +1,346 @@ # policyengine-observability -Shared PolicyEngine observability runtime for fail-open local timings, -structured logs, OpenTelemetry traces, and OpenTelemetry metrics. +`policyengine-observability` provides a shared runtime for structured logs, +OpenTelemetry traces and metrics, request propagation, and framework request +instrumentation. Each application owns its service identity, attribute policy, +logging destinations, OTLP endpoints, and credentials. -The package intentionally keeps framework support in adapters: +Version 2 uses an explicitly owned runtime. `configure(config)` returns that +runtime, and every adapter or manual operation receives it. Configuration does +not select a destination from the deployment platform or contact a remote +service. -- `policyengine_observability.adapters.flask` -- `policyengine_observability.adapters.fastapi` -- `policyengine_observability.integrations.httpx` +## Install -OpenTelemetry support is installed and enabled by default. Timing and -structured logging run even without an OTLP collector; when no endpoint is -configured, spans and metrics stay in-process while logs still receive trace -context. Set `OTEL_ENABLED=false` to opt out. Configure -`OTEL_EXPORTER_OTLP_ENDPOINT` to export traces and metrics. +Install only the integrations used by the service: -## Log routing profiles +```bash +pip install "policyengine-observability[otel,otlp-grpc]" +pip install "policyengine-observability[flask,httpx,google]" +pip install "policyengine-observability[fastapi,httpx,google]" +``` -Log routing is owned by this package: consumers set one env var and the -package expands it into destinations, formats, and transport. +The base package has no required dependencies. OpenTelemetry, OTLP exporters, +Google authentication and logging, and web frameworks are optional extras and +are imported only when configured or called. + +## Configure a runtime + +Service and deployment identity are explicit. Logging and OTel use separate +configuration so they can write to different stores. + +```python +from policyengine_observability import ( + DeploymentIdentity, + LoggingConfig, + ObservabilityConfig, + OTelConfig, + OTLPExporterConfig, + ServiceIdentity, + StdoutLogDestination, + configure, +) + +config = ObservabilityConfig( + service=ServiceIdentity( + name="example-api", + namespace="policyengine.example", + version="1.2.3", + role="api", + ), + deployment=DeploymentIdentity( + environment="production", + platform="google_cloud_run", + region="us-central1", + ), + logging=LoggingConfig( + destinations=(StdoutLogDestination(),), + ), + otel=OTelConfig( + traces=OTLPExporterConfig(endpoint="collector:4317"), + metrics=OTLPExporterConfig(endpoint="collector:4317"), + ), + application_attribute_keys=frozenset({"country_id", "backend"}), + dispatch_attribute_keys=frozenset({"job_id", "run_id"}), +) +runtime = configure(config) +``` -```bash -OBSERVABILITY_LOG_PROFILE=gcp-agent # or gcp-direct | plain-sync | auto -``` - -- `gcp-agent` — google-format stdout only, for platforms whose logging - agent ingests stdout (Cloud Run, GKE). Fully synchronous, zero - threads; the agent ships lines to Cloud Logging with severity, trace, - span, and labels promoted to first-class LogEntry fields. -- `gcp-direct` — plain stdout (the durable record) plus queued direct - Cloud Logging writes, for platforms with no ingesting agent (Modal). - Requires a resolvable Google Cloud project; downgrades to `plain-sync` - with a warning otherwise. -- `plain-sync` — plain stdout only, guaranteed zero threads. Local - development and the kill switch: setting it disables all background - log machinery. -- `auto` (default) — detects the platform via `OBSERVABILITY_PLATFORM` - (`google_cloud_run`/`modal`), then `K_SERVICE`, then Modal env - markers; when nothing matches, caller-supplied defaults apply. - -The granular controls still exist underneath and override the profile's -expansion when set explicitly: +`configure` validates the complete configuration before it creates workers, +exporters, or logging handlers. Invalid values raise `ConfigurationError` with +the fields that must be corrected. Unavailable credentials or destinations +after successful validation remain nonfatal runtime failures. + +The default logging destination is one-line JSON on standard output. An OTel +runtime without an exporter still creates local trace context for log +correlation. It does not send traces or metrics remotely. + +## Logging destinations + +Application code emits a provider-neutral record. `LoggingConfig` selects one +or more destination strategies when the runtime starts. + +```python +from policyengine_observability import ( + GoogleCloudLogDestination, + GoogleCloudLogFormatter, +) + +logging = LoggingConfig( + destinations=( + StdoutLogDestination( + formatter=GoogleCloudLogFormatter("trace-project"), + ), + GoogleCloudLogDestination( + project_id="logging-project", + log_name="example-api", + queue_capacity=1_000, + batch_size=100, + write_timeout_seconds=5, + ), + ), +) +``` -```bash -OBSERVABILITY_LOG_DESTINATIONS=stdout,google_cloud_logging -OBSERVABILITY_STDOUT_FORMAT=google -OBSERVABILITY_GOOGLE_CLOUD_PROJECT=policyengine-api -OBSERVABILITY_GOOGLE_CLOUD_LOG_NAME=policyengine-observability -``` - -Destinations are named strategies: `stdout` is `inline` (synchronous on -the caller's thread), and every `remote` strategy — `google_cloud_logging` -today; future backends register the same way — is automatically wrapped -in the queued transport below. Google Cloud Logging uses Application -Default Credentials and requires permission to create log entries, -typically through `roles/logging.logWriter`. - -Backends plug in through two top-level hooks, `register_destination` -(with `transport="inline"|"remote"` and an optional `required_config` -tuple naming the config fields the strategy needs — profiles that name -the strategy downgrade gracefully when one is missing) and -`register_stdout_formatter`. Registration happens at import time, so an -external backend module must be imported before observability is -configured. Backend-specific knobs are the strategy's own: the Google -strategy reads `OBSERVABILITY_GOOGLE_WRITE_TIMEOUT_SECONDS` itself at -construction (so it is re-read on `restart_observability()`) rather -than through a core config field. Name lookups for destinations, -formatters, and profiles all forgive case, whitespace, and -hyphen/underscore variance; an unknown format name falls back to plain -and is reported through the internal-error channel, and a registered -formatter factory that raises degrades to the built-in plain formatter -the same way rather than breaking configuration. - -## Log emission and delivery semantics - -Remote destinations never write on a request thread. The log call only -snapshots the payload, stamps the enqueue time, and appends to a bounded -in-memory queue (microseconds, never blocks, never raises); a stdlib -`QueueListener` thread drains the queue and performs the writes, sending -the enqueue time as the entry timestamp so delayed writes keep their -event time. - -Delivery through the queue is best-effort by design — stdout is the -durable sibling record. When the queue is full the newest record is -dropped, and drops are counted and reported through the internal-error -channel (first drop, then every 100th). Write failures are likewise -reported and the record dropped; there is deliberately no breaker or -retry queue in the transport. Each Google write carries an explicit -budget that caps the call and its transient-error retries: +The formatter adds Cloud Logging trace-correlation fields only to the output +it formats. The canonical record retains the portable `trace_id`, `span_id`, +and `trace_sampled` fields. Each destination and formatter receives a deep +copy of the canonical record, so mutations to nested values remain local to +that destination. + +Configured sensitive values are replaced in messages, exception details, and +allowlisted string attributes before length limits are applied and before the +record reaches a logging destination, span, or metric exporter. This also +applies to exception events on spans and local internal diagnostics. + +Every queued destination has its own bounded queue and worker. A blocked or +failing destination cannot delay another destination or the application +operation that emitted the record. Queue saturation drops the newest record +and records a local diagnostic. + +### Custom destinations + +Use `CustomLogDestination` for a writer owned by an application or another +package: + +```python +from policyengine_observability import CustomLogDestination + +logging = LoggingConfig( + destinations=( + CustomLogDestination( + name="internal-log-store", + writer_factory=lambda: InternalLogWriter(), + delivery="queued", + ), + ), +) +``` -```bash -OBSERVABILITY_GOOGLE_WRITE_TIMEOUT_SECONDS=10.0 -OBSERVABILITY_LOG_QUEUE_MAXSIZE=1000 -OBSERVABILITY_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS=2.0 -``` - -(The bounded write rebinds the client's private gapic method at -construction; when that handle is unavailable — HTTP transports, -injected fakes — the library's ~60s default applies, which is harmless -off the request path. All numeric knobs are clamped, so `0`, negative, -or non-finite values can never disable or unbound a mechanism. The -Google client library's one-time instrumentation diagnostic entry is -suppressed at construction, so the stream carries only the records the -service asked to write.) - -Shutdown closes log destinations inside the same bounded budget that -flushes OpenTelemetry (`OBSERVABILITY_SHUTDOWN_TIMEOUT_SECONDS`); a -queue that cannot drain before its deadline is abandoned with a report. -A hard kill loses whatever was still queued. Note the google stdout -format sets no `time` key: stdout emission is synchronous, so the -agent's receive time is the correct event time. - -Processes that fork or restore from memory snapshots do not preserve -threads or network clients. Call `restart_observability()` from the -post-restore or post-fork hook (for example gunicorn `post_fork` when -using `--preload`, or a Modal post-snapshot hook) — it closes and -rebuilds destinations from configuration, and must only be called from -single-threaded lifecycle moments, before serving traffic. It is a -no-op when observability is disabled, so the kill switch holds across -forks and restores. - -Request and operation logs include two timing views: - -- `timings_ms` and `timing_counts` are flat inclusive aggregates by segment - name, intended for quick scanning and compatibility with existing log - queries. -- `segment_tree` is an ordered nested view of segment occurrences. Repeated - sibling segments are preserved as separate entries, and safe scalar segment - attributes are included so callers can distinguish settings such as - `simulation_kind=baseline` versus `simulation_kind=reform`. - -Core structured log fields take precedence over caller-provided attributes with -the same keys. - -On runtimes that do not provide Application Default Credentials, set -`GCP_CREDENTIALS_JSON` to a service account JSON document. The Google Cloud -Logging destination will materialize it into a temporary credentials file and -pass those credentials directly to the Google client. If the credential -bootstrap fails, observability fails open and continues without raising into -application code. - -Prefer OIDC-based Workload Identity Federation over long-lived service account -keys when the runtime can provide an OIDC subject token. Modal injects -generated identity tokens into Function containers through -`MODAL_IDENTITY_TOKEN`; other runtimes can provide -`OBSERVABILITY_GOOGLE_OIDC_TOKEN`. The runtime needs these values: +The writer implements `write(record)`. It may optionally implement +`write_many(records)` and `close()`. A separate integration package can also +provide a class implementing `LogDestinationStrategy`. Network writers should +always use `delivery="queued"`. Cleanup for inline writers runs on daemon +threads, and orderly shutdown waits for it only within the configured logging +shutdown timeout. + +## OTLP destinations and authentication + +Traces and metrics have independent exporter configurations: + +```python +otel = OTelConfig( + traces=OTLPExporterConfig( + endpoint="https://trace-collector.example", + protocol="http/protobuf", + headers=(("x-api-key", "trace-key"),), + ), + metrics=OTLPExporterConfig( + endpoint="metrics-collector.example:4317", + protocol="grpc", + headers=(("x-api-key", "metric-key"),), + ), +) +``` + +For OTLP over HTTP, the default `endpoint_mode="base"` appends +`/v1/traces` or `/v1/metrics` to the configured endpoint. Set +`endpoint_mode="signal"` when the endpoint already identifies the exact +signal route. + +For a Google ID-token protected collector, select the authentication strategy +explicitly: + +```python +from policyengine_observability import GoogleIdTokenAuth + +traces = OTLPExporterConfig( + endpoint="collector.example:443", + auth=GoogleIdTokenAuth("https://collector.example"), +) +``` + +`ObservabilityConfig.from_env` reads the standard common and signal-specific +OTel settings, including: ```bash -OBSERVABILITY_GOOGLE_OIDC_TOKEN=OIDC_TOKEN_FROM_RUNTIME -OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER=projects/PROJECT_NUMBER/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID -OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL=observability-writer@PROJECT_ID.iam.gserviceaccount.com -OBSERVABILITY_GOOGLE_CLOUD_PROJECT=PROJECT_ID -``` - -When `MODAL_IDENTITY_TOKEN` or `OBSERVABILITY_GOOGLE_OIDC_TOKEN` is present -alongside `OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER`, the Google Cloud -Logging destination writes a temporary external-account credential -configuration and passes those credentials directly to the Cloud Logging -client. If `OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL` is present, the -configuration uses service account impersonation. This keeps observability -credentials separate from any application-level `GOOGLE_APPLICATION_CREDENTIALS` -or `GCP_CREDENTIALS_JSON` used by the service for other Google clients. - -The Google Cloud setup needs: - -- A Workload Identity Pool and OIDC provider whose issuer matches Modal's OIDC - issuer, `https://oidc.modal.com`. -- Attribute mapping for the Modal token claims you want to authorize, such as - `google.subject=assertion.sub`. -- A service account with `roles/logging.logWriter` on the log project. -- An IAM binding granting the workload identity principal - `roles/iam.workloadIdentityUser` on that service account. - -For the fixed PolicyEngine Google Cloud destination, see -[`docs/operations/google-cloud-stage3-runbook.md`](docs/operations/google-cloud-stage3-runbook.md). +OTEL_EXPORTER_OTLP_ENDPOINT=collector.example:4317 +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=trace-collector.example:4317 +OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=metric-collector.example:4317 +OTEL_EXPORTER_OTLP_PROTOCOL=grpc +OTEL_EXPORTER_OTLP_TRACES_HEADERS=x-api-key=trace-key +OTEL_EXPORTER_OTLP_METRICS_HEADERS=x-api-key=metric-key +``` + +The common HTTP endpoint is treated as a base URL. Signal-specific HTTP +endpoint variables are passed to the exporter exactly as configured, following +the OpenTelemetry environment-variable contract. + +`POLICYENGINE_OTEL_GOOGLE_AUDIENCE` selects `GoogleIdTokenAuth` for both +signals. `POLICYENGINE_OTEL_TRACES_GOOGLE_AUDIENCE` and +`POLICYENGINE_OTEL_METRICS_GOOGLE_AUDIENCE` override it per signal. + +Google authentication constructed from `GCP_CREDENTIALS_JSON`, +`MODAL_IDENTITY_TOKEN`, or `OBSERVABILITY_GOOGLE_OIDC_TOKEN` stays in process +memory. The package passes credential data and subject tokens directly to +Google Auth and does not create temporary credential files. A path explicitly +provided through `GOOGLE_APPLICATION_CREDENTIALS` remains supported. + +Applications can share a collector by configuring the same endpoint and +credentials. Another application can use a separate collector or any +OTLP-compatible service without changing this package. + +## Flask and FastAPI + +Install a framework adapter once while constructing the application: + +```python +from flask import Flask +from policyengine_observability import instrument_flask + +app = Flask(__name__) +instrument_flask(app, runtime) +``` + +```python +from fastapi import FastAPI +from policyengine_observability import instrument_fastapi + +app = FastAPI() +instrument_fastapi(app, runtime) +``` + +Both adapters extract W3C trace context and +`X-PolicyEngine-Request-Id`, create one server span, emit one completion +record, record request metrics, and clear context-local state. Repeated calls +reuse the first runtime associated with the application. Install the Flask +adapter before serving the first request. If Flask rejects callback +registration, the adapter restores the prior callback registries, leaves the +application unmarked, reports a local diagnostic, and returns without raising. + +## Application instrumentation + +The package instruments framework and transport mechanics. Applications own +the names and boundaries of their domain operations. + +```python +with runtime.operation( + "simulation.run", + attributes={"backend": "modal"}, +): + with runtime.span("simulation.build"): + simulation = build_simulation() + result = calculate(simulation) +``` + +Operations and spans support synchronous and asynchronous context management +and decoration: + +```python +@runtime.span("simulation.calculate") +async def calculate(simulation): + return await simulation.calculate() +``` + +Function arguments and return values are never captured. The runtime accepts +only application-configured attribute keys and scalar values. + +```python +runtime.set_context(auth_result="accepted", simulation_id="sim-123") +runtime.event("simulation.dispatched", attributes={"backend": "modal"}) + +try: + load_result() +except ValueError as error: + runtime.record_exception(error, handled=True) +``` + +## Python logging and HTTP propagation + +Instrument a specific Python logger or configure root capture explicitly: + +```python +import logging +from policyengine_observability import instrument_logging + +logger = logging.getLogger("policyengine.example") +instrument_logging(logger, runtime) +``` + +Only the supplied HTTPX client is modified: + +```python +import httpx +from policyengine_observability import instrument_httpx + +client = httpx.AsyncClient() +instrument_httpx(client, runtime) +``` + +The request hook injects active W3C context and the PolicyEngine request ID. +Other clients in the process remain unchanged. + +For asynchronous dispatch, serialize the bounded correlation context with the +job request and restore it around the worker operation: + +```python +request.observability_context = runtime.capture_context() + +with runtime.operation( + "simulation.run", + remote_context=request.observability_context, +): + return run_simulation(request) +``` + +## Process restoration and shutdown + +After a process image or memory snapshot is restored, rebuild process-local +locks, context, queues, threads, credentials, and exporters before accepting +work: + +```python +@modal.enter(snap=False) +def restore_process_state(self): + self.runtime.restart_after_snapshot() +``` + +Call `runtime.shutdown()` during orderly process shutdown. Logging and OTel +each use their own timeout. After configuration validation succeeds, missing +optional dependencies, credential failures, unavailable destinations, queue +saturation, exporter errors, and shutdown timeouts produce rate-limited +diagnostics on standard error. These runtime failures do not change application +responses, return values, or exceptions. ## Release workflow -Changes should include a Towncrier fragment in `changelog.d/`. Pull requests -run changelog, Ruff, and coverage checks. Pushes to `main` run the same gates, -then publish a versioning commit that builds the changelog and bumps -`pyproject.toml`. That versioning commit publishes the package to PyPI through -trusted publishing, creates a matching git tag, and opens a GitHub release. +Changes include a Towncrier fragment in `changelog.d/`. Pull requests run Ruff, +tests, type checks, and coverage checks. The release workflow builds +distributions and publishes through PyPI trusted publishing. ## License -Code in this repository is released under the [MIT License](LICENSE). Original text and figures are released under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) with attribution to PolicyEngine. Third-party data and materials keep their own terms. +Code in this repository is released under the [MIT License](LICENSE). Original +text and figures are released under +[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) with attribution to +PolicyEngine. Third-party materials retain their terms. diff --git a/changelog.d/29.breaking.md b/changelog.d/29.breaking.md new file mode 100644 index 0000000..4864bc3 --- /dev/null +++ b/changelog.d/29.breaking.md @@ -0,0 +1,29 @@ +Replace the implicit global runtime and version 1 decorators with an explicit +runtime returned by `configure`. Version 2 adds runtime-bound operations and +spans, idempotent Flask and FastAPI adapters, explicit HTTPX and Python logging +instrumentation, provider-neutral logging destination strategies, independent +trace and metric OTLP exporters, pluggable exporter authentication, and +allowlisted cross-service correlation. Every remote logging destination uses +an independent bounded queue, and logging and OTel failures remain isolated +from application responses and calculations. Consumers must replace removed +version 1 imports and provide their own identity, destinations, attribute +policy, endpoints, and credentials before upgrading. +`configure` validates these values before runtime setup and raises +`ConfigurationError` for invalid configuration. +Inline writer cleanup now runs outside application execution and respects the +configured logging shutdown timeout. Google credential JSON and workload +identity subject tokens are passed to Google Auth in memory instead of being +written to temporary files. +Configured sensitive values are redacted from allowlisted string attributes, +including values sent to logs, spans, and metrics. Each logging destination and +formatter now receives a deep copy so nested mutations cannot affect another +destination or the canonical record. +Invalid `sensitive_values` entries now fail configuration validation before +runtime setup. Flask callback installation failures restore the prior +application state and do not propagate into application execution. +Configured sensitive values are also redacted from OpenTelemetry exception +events and local diagnostics. Attribute allowlists now require frozensets of +non-empty strings. Flask instrumentation starts before existing request +callbacks and completes during request teardown so failures retain exception +details. HTTP 5xx responses set span error status. Signal-specific OTLP/HTTP +environment endpoints are passed through unchanged. diff --git a/docs/engineering/skills/repository-guidance.md b/docs/engineering/skills/repository-guidance.md index 872d237..f17df7c 100644 --- a/docs/engineering/skills/repository-guidance.md +++ b/docs/engineering/skills/repository-guidance.md @@ -25,19 +25,18 @@ uv run --extra dev towncrier check --compare-with origin/main - `policyengine_observability/config.py` resolves environment-driven runtime configuration. -- `policyengine_observability/context.py` defines request and operation log - payload structures. - `policyengine_observability/runtime.py` preserves the public runtime API, configures the components, and coordinates their shutdown. -- `policyengine_observability/_state.py` owns shared context variables. -- `policyengine_observability/_operations.py` and `_requests.py` manage - operation and request lifecycles, respectively. -- `policyengine_observability/segments.py` manages segment naming, nesting, - and timing. -- `policyengine_observability/logging.py` emits structured logs and records - observability failures without interrupting application operations. -- `policyengine_observability/_metrics.py` and `_tracing.py` record metrics - and manage OpenTelemetry traces, respectively. +- `policyengine_observability/delivery.py` isolates configured log + destinations and gives each remote destination its own bounded queue. +- `policyengine_observability/destinations/` contains the provider-neutral + destination strategy contract and optional built-in destinations. +- `policyengine_observability/schema.py` builds provider-neutral structured + records. Provider fields belong in destination formatters. +- `policyengine_observability/otel.py` records metrics and traces and exports + each enabled signal through its configured OTLP transport. +- `policyengine_observability/google_auth.py` and + `google_credentials.py` contain optional Google authentication strategies. - `policyengine_observability/adapters/` contains framework adapters such as Flask and FastAPI. - `policyengine_observability/integrations/` contains optional integrations @@ -54,21 +53,26 @@ uv run --extra dev towncrier check --compare-with origin/main CLI scripts, and tests. - Keep OpenTelemetry optional and lazily imported. Timing and structured logging must work without an OTel backend. -- Observability failures must fail open: record an internal observability error - when practical, but do not break the application operation being observed. +- Keep canonical records free of provider-specific field names. Apply those + fields in the configured destination formatter or writer. +- Keep OTLP transport independent from exporter authentication and allow + traces and metrics to use different endpoints. +- Reject invalid configuration before starting runtime components. After + successful validation, record runtime failures internally when practical + without breaking the application operation being observed. - Preserve structured log schemas. Make additive changes when possible; bump schema versions for breaking payload changes. - Keep metric attributes bounded and low-cardinality. Do not put raw paths, full URLs, request bodies, or unbounded user-provided values into metric labels. -- Keep segment names stable. Prefer registered segment enums in consuming +- Keep span names stable. Prefer registered span enums in consuming applications, while preserving safe string fallback behavior. ## Testing Add focused tests for context behavior and failure paths whenever changing the runtime or its components. The corresponding `tests/test_runtime_*.py` -modules cover operations, requests, segments, log emission, and tracing. +modules cover operations, requests, spans, log emission, and tracing. Adapter changes should include framework-level tests that exercise request setup, response headers, error paths, and teardown behavior. diff --git a/docs/operations/google-cloud-stage3-runbook.md b/docs/operations/google-cloud-stage3-runbook.md deleted file mode 100644 index c6c25b2..0000000 --- a/docs/operations/google-cloud-stage3-runbook.md +++ /dev/null @@ -1,195 +0,0 @@ -# Google Cloud Stage 3 Observability Runbook - -This runbook documents the fixed Google Cloud Logging destination for -PolicyEngine observability records. - -The public runbook intentionally uses placeholders for live project IDs, -project numbers, service accounts, workload identity provider paths, and sink -writer identities. Keep concrete values in private infrastructure state, -repository/environment secrets, or internal operations documentation. - -## Central Destination - -- Project: `` -- Project number: `` -- Log bucket: `` -- Bucket location: `` -- Bucket retention: 30 days -- Log Analytics: enabled -- Primary log name: `` - -Required APIs: - -```bash -gcloud services enable \ - logging.googleapis.com \ - iamcredentials.googleapis.com \ - sts.googleapis.com \ - --project= -``` - -## Cloud Run Logging - -Cloud Run has two log paths. - -App observability records are written directly by the observability package to -the central project. Cloud Run runtime service accounts need: - -```text -roles/logging.logWriter -``` - -on ``. - -Service accounts should be granted explicitly per deployed service and -environment: - -```text - - -``` - -Native Cloud Run logs are routed with a Cloud Logging sink from -the source application project: - -```text -sink: -destination: logging.googleapis.com/projects//locations//buckets/ -filter: resource.type="cloud_run_revision" -writer: -``` - -The sink writer has `roles/logging.bucketWriter` on -``. - -## Modal Logging - -Modal platform logs stay in Modal for now. Modal simulation container -observability records are written directly to Google Cloud Logging by the -observability package. - -The Modal writer service account is an environment-specific service account: - -```text - -``` - -It has `roles/logging.logWriter` on ``. - -Modal Workload Identity Federation: - -```text -provider: projects//locations/global/workloadIdentityPools//providers/ -issuer: https://oidc.modal.com -audience: oidc.modal.com -``` - -The writer service account grants `roles/iam.workloadIdentityUser` to: - -```text -principalSet://iam.googleapis.com/projects//locations/global/workloadIdentityPools// -``` - -The impersonation grant must be constrained by Modal OIDC claims or by an -equivalent provider condition. Do not use an unconstrained wildcard grant. The -condition should allow only the expected Modal app names and environments for -the services being deployed. - -## Runtime Configuration - -Use these variables for deployed Cloud Run and Modal environments: - -```bash -# Cloud Run: the agent ingests stdout, no direct writes needed. -OBSERVABILITY_LOG_PROFILE=gcp-agent -OBSERVABILITY_GOOGLE_CLOUD_PROJECT= - -# Modal: stdout plus queued direct Cloud Logging writes. -OBSERVABILITY_LOG_PROFILE=gcp-direct -OBSERVABILITY_GOOGLE_CLOUD_PROJECT= -OBSERVABILITY_GOOGLE_CLOUD_LOG_NAME= -OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER=projects//locations/global/workloadIdentityPools//providers/ -OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL= -``` - -Cloud Run does not need the Modal WIF variables. Modal does. - -Do not use `OBSERVABILITY_GOOGLE_LOG_NAME`; the package reads -`OBSERVABILITY_GOOGLE_CLOUD_LOG_NAME`. - -## Smoke Tests - -Write a manual structured log: - -```bash -gcloud logging write \ - '{"event":"stage3_smoke_test","service_name":"","schema_version":"policyengine.observability.smoke.v1"}' \ - --payload-type=json \ - --project= \ - --severity=INFO -``` - -Read it back: - -```bash -gcloud logging read \ - 'logName="projects//logs/" AND jsonPayload.event="stage3_smoke_test"' \ - --project= \ - --limit=1 \ - --format=json -``` - -Find app observability logs: - -```text -logName="projects//logs/" -jsonPayload.service_name="" -``` - -Find Cloud Run app observability logs: - -```text -jsonPayload.service_name="" -jsonPayload.platform="google_cloud_run" -``` - -Find Modal simulation container observability logs: - -```text -jsonPayload.service_name="" -jsonPayload.platform="modal" -jsonPayload.service_role="modal_worker" -``` - -Find native Cloud Run logs routed through the sink: - -```text -resource.type="cloud_run_revision" -``` - -Find observability failures: - -```text -jsonPayload.event="observability_internal_error" -``` - -## Rollback - -For a failing runtime, set the kill switch — plain stdout only, with all -background log machinery disabled: - -```bash -OBSERVABILITY_LOG_PROFILE=plain-sync -``` - -To restore the temporary bridge destination, set: - -```bash -OBSERVABILITY_GOOGLE_CLOUD_PROJECT= -OBSERVABILITY_GOOGLE_CLOUD_LOG_NAME= -OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER= -OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL= -``` - -Leave the central project and sink in place during rollback unless the sink -itself is the source of the problem. diff --git a/policyengine_observability/__init__.py b/policyengine_observability/__init__.py index 538d1f0..c012bc0 100644 --- a/policyengine_observability/__init__.py +++ b/policyengine_observability/__init__.py @@ -1,192 +1,64 @@ -from __future__ import annotations - -from typing import Any - -from .config import ObservabilityConfig -from .context import OperationObservabilityContext, RequestObservabilityContext -from .destinations import register_destination, register_stdout_formatter -from .destinations.google_credentials import ( - configure_google_application_credentials, - load_google_credentials, +"""PolicyEngine observability version 2 public interface.""" + +from .adapters import instrument_fastapi, instrument_flask +from .config import ( + ConfigurationError, + DeploymentIdentity, + LoggingConfig, + ObservabilityConfig, + OTelConfig, + OTLPAuthentication, + OTLPExporterConfig, + ServiceIdentity, + TelemetryLimits, ) +from .destinations import ( + CustomLogDestination, + GoogleCloudLogDestination, + GoogleCloudLogFormatter, + LogDestinationStrategy, + RecordWriter, + StdoutLogDestination, +) +from .google_auth import GoogleIdTokenAuth +from .integrations import instrument_httpx from .runtime import ( - OBSERVABILITY_INTERNAL_DISPATCH_HEADER, REQUEST_ID_HEADER, TRACEPARENT_HEADER, + TRACESTATE_HEADER, + ObservabilityLogHandler, ObservabilityRuntime, - observability_runtime, - set_observability_runtime, + configure, + instrument_logging, ) -from .segments import UNKNOWN_SEGMENT, coerce_segment_name - - -def current_context() -> RequestObservabilityContext | None: - return observability_runtime().current_context() - - -def current_operation() -> OperationObservabilityContext | None: - return observability_runtime().current_operation() - - -def set_attribute(key: str, value: Any) -> None: - observability_runtime().set_attribute(key, value) - - -def record_error( - exc: BaseException, - *, - handled: bool, - status_code: int | None = None, - include_stack: bool = True, -) -> None: - observability_runtime().record_error( - exc, - handled=handled, - status_code=status_code, - include_stack=include_stack, - ) - - -def record_event(event: str, **fields: Any) -> None: - observability_runtime().record_event(event, **fields) - - -def traceparent_header() -> str | None: - return observability_runtime().traceparent_header() - - -def capture_context(): - return observability_runtime().capture_context() - - -def mark(key: str, ms: float) -> None: - observability_runtime().mark(key, ms) - - -def mark_ttft(key: str = "ttft_ms") -> None: - observability_runtime().mark_ttft(key) - - -def mark_ttft_attribute(key: str = "ttft_ms") -> None: - observability_runtime().mark_ttft_attribute(key) - - -def start_scope( - timings: dict[str, float], - *, - name: str = "operation", - parent_context: Any = None, - **attrs: Any, -): - return observability_runtime().start_scope( - timings, - name=name, - parent_context=parent_context, - **attrs, - ) - - -def annotate(handle=None, **attrs: Any) -> None: - observability_runtime().annotate(handle, **attrs) - - -def end_scope(handle, error: BaseException | None = None) -> None: - observability_runtime().end_scope(handle, error) - - -def instrument_fastapi(app: Any) -> None: - observability_runtime().instrument_fastapi(app) - - -def instrument_httpx() -> None: - observability_runtime().instrument_httpx() - - -def shutdown_observability() -> None: - observability_runtime().shutdown() - - -def shutdown_tracing() -> None: - shutdown_observability() - - -def restart_observability() -> None: - """Close and rebuild log destinations from configuration. - - For runtimes whose processes fork or restore from memory snapshots - (threads and network clients do not survive either). Call ONLY from - single-threaded lifecycle moments — a post-snapshot-restore hook, a - post-fork hook, before serving traffic. - """ - observability_runtime().restart_log_destinations() - - -def operation(name: str, *, flavor: str | None = None, **attrs: Any): - return observability_runtime().operation(name, flavor=flavor, **attrs) - - -def entrypoint( - name: str | None = None, - *, - flavor: str | None = None, - **attrs: Any, -): - return observability_runtime().entrypoint( - name, - flavor=flavor, - **attrs, - ) - - -def segment(name: Any, **attrs: Any): - return observability_runtime().segment(name, **attrs) - - -def asegment(name: Any, **attrs: Any): - return observability_runtime().asegment(name, **attrs) - - -def collect_timings(name: str = "operation", **attrs: Any): - return observability_runtime().collect_timings(name, **attrs) - +from .schema import SCHEMA_VERSION __all__ = [ - "OBSERVABILITY_INTERNAL_DISPATCH_HEADER", "REQUEST_ID_HEADER", + "SCHEMA_VERSION", "TRACEPARENT_HEADER", - "UNKNOWN_SEGMENT", - "OperationObservabilityContext", + "TRACESTATE_HEADER", + "CustomLogDestination", + "ConfigurationError", + "DeploymentIdentity", + "GoogleCloudLogDestination", + "GoogleCloudLogFormatter", + "GoogleIdTokenAuth", + "LogDestinationStrategy", + "LoggingConfig", + "OTLPAuthentication", + "OTLPExporterConfig", + "OTelConfig", "ObservabilityConfig", + "ObservabilityLogHandler", "ObservabilityRuntime", - "RequestObservabilityContext", - "annotate", - "asegment", - "capture_context", - "coerce_segment_name", - "collect_timings", - "configure_google_application_credentials", - "load_google_credentials", - "current_context", - "current_operation", - "end_scope", - "entrypoint", + "RecordWriter", + "ServiceIdentity", + "TelemetryLimits", + "StdoutLogDestination", + "configure", "instrument_fastapi", + "instrument_flask", "instrument_httpx", - "mark", - "mark_ttft", - "mark_ttft_attribute", - "observability_runtime", - "operation", - "record_error", - "record_event", - "register_destination", - "register_stdout_formatter", - "restart_observability", - "segment", - "set_attribute", - "set_observability_runtime", - "shutdown_observability", - "shutdown_tracing", - "start_scope", - "traceparent_header", + "instrument_logging", ] diff --git a/policyengine_observability/_metrics.py b/policyengine_observability/_metrics.py deleted file mode 100644 index ac2d420..0000000 --- a/policyengine_observability/_metrics.py +++ /dev/null @@ -1,188 +0,0 @@ -"""Metric instrument creation and recording.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from .runtime import ObservabilityRuntime - - -class _NoOpInstrument: - def add(self, *_args, **_kwargs) -> None: - return None - - def record(self, *_args, **_kwargs) -> None: - return None - - -class MetricRecorder: - def __init__(self, runtime: ObservabilityRuntime) -> None: - self.runtime = runtime - - def record_operation_metric( - self, - duration_seconds: float, - attributes: dict[str, str], - ) -> None: - try: - self.runtime.operation_duration.record( - duration_seconds, attributes - ) - self.runtime.operations.add(1, attributes) - except BaseException as exc: - self.runtime.log_observability_failure( - "metrics.record_operation", exc - ) - - def record_request_metric( - self, - duration_seconds: float, - attributes: dict[str, str], - ) -> None: - try: - self.runtime.http_duration.record(duration_seconds, attributes) - self.runtime.requests.add(1, attributes) - except BaseException as exc: - self.runtime.log_observability_failure( - "metrics.record_request", exc - ) - - def record_segment_metric( - self, - segment: str, - duration_seconds: float, - attributes: dict[str, str], - *, - backend_segment: bool = False, - ) -> None: - try: - segment_attributes = {**attributes, "segment": segment} - self.runtime.segment_duration.record( - duration_seconds, segment_attributes - ) - if segment == "calculation": - self.runtime.calculate_duration.record( - duration_seconds, attributes - ) - if backend_segment: - self.runtime.backend_duration.record( - duration_seconds, - segment_attributes, - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "metrics.record_segment", - exc, - segment=segment, - ) - - def record_error_metric(self, attributes: dict[str, str]) -> None: - try: - self.runtime.errors.add(1, attributes) - except BaseException as exc: - self.runtime.log_observability_failure("metrics.record_error", exc) - - def record_rate_limited_metric(self, attributes: dict[str, str]) -> None: - try: - self.runtime.rate_limited.add(1, attributes) - except BaseException as exc: - self.runtime.log_observability_failure( - "metrics.record_rate_limited", exc - ) - - def record_failover_event_metric(self, attributes: dict[str, str]) -> None: - try: - self.runtime.failover_events.add(1, attributes) - except BaseException as exc: - self.runtime.log_observability_failure( - "metrics.record_failover_event", - exc, - ) - - def record_active_request( - self, - delta: int, - attributes: dict[str, str], - ) -> None: - try: - self.runtime.active_requests.add(delta, attributes) - except BaseException as exc: - self.runtime.log_observability_failure( - "metrics.add_active_request", exc - ) - - def _configure_instruments(self) -> None: - self.runtime.operation_duration = self.runtime._instrument( - getattr(self.runtime.meter, "create_histogram", None), - "policyengine.operation.duration", - unit="s", - description="PolicyEngine operation duration.", - ) - self.runtime.http_duration = self.runtime._instrument( - getattr(self.runtime.meter, "create_histogram", None), - "http.server.request.duration", - unit="s", - description="HTTP server request duration.", - ) - self.runtime.segment_duration = self.runtime._instrument( - getattr(self.runtime.meter, "create_histogram", None), - "policyengine.segment.duration", - unit="s", - description="PolicyEngine operation segment duration.", - ) - self.runtime.calculate_duration = self.runtime._instrument( - getattr(self.runtime.meter, "create_histogram", None), - "policyengine.calculate.duration", - unit="s", - description="PolicyEngine calculate operation duration.", - ) - self.runtime.backend_duration = self.runtime._instrument( - getattr(self.runtime.meter, "create_histogram", None), - "policyengine.backend.duration", - unit="s", - description="PolicyEngine backend call duration.", - ) - self.runtime.operations = self.runtime._instrument( - getattr(self.runtime.meter, "create_counter", None), - "policyengine.operations", - description="PolicyEngine operation count.", - ) - self.runtime.requests = self.runtime._instrument( - getattr(self.runtime.meter, "create_counter", None), - "policyengine.requests", - description="PolicyEngine request count.", - ) - self.runtime.errors = self.runtime._instrument( - getattr(self.runtime.meter, "create_counter", None), - "policyengine.errors", - description="PolicyEngine error count.", - ) - self.runtime.rate_limited = self.runtime._instrument( - getattr(self.runtime.meter, "create_counter", None), - "policyengine.rate_limited_requests", - description="PolicyEngine rate-limited request count.", - ) - self.runtime.failover_events = self.runtime._instrument( - getattr(self.runtime.meter, "create_counter", None), - "policyengine.failover.events", - description="PolicyEngine failover event count.", - ) - self.runtime.active_requests = self.runtime._instrument( - getattr(self.runtime.meter, "create_up_down_counter", None), - "http.server.active_requests", - description="Active HTTP server requests.", - ) - - def _instrument(self, factory, *args, **kwargs): - if factory is None: - return _NoOpInstrument() - try: - return factory(*args, **kwargs) - except BaseException as exc: - self.runtime.log_observability_failure( - "metrics.create_instrument", - exc, - instrument=args[0] if args else None, - ) - return _NoOpInstrument() diff --git a/policyengine_observability/_operations.py b/policyengine_observability/_operations.py deleted file mode 100644 index 9cefa7b..0000000 --- a/policyengine_observability/_operations.py +++ /dev/null @@ -1,428 +0,0 @@ -"""Operation lifecycle and timing-scope behavior.""" - -from __future__ import annotations - -import inspect -import time -from contextlib import contextmanager -from functools import wraps -from typing import TYPE_CHECKING, Any - -from . import _state -from .context import ( - ErrorRecord, - OperationObservabilityContext, -) - -if TYPE_CHECKING: - from .runtime import ObservabilityRuntime - - -class OperationLifecycle: - def __init__(self, runtime: ObservabilityRuntime) -> None: - self.runtime = runtime - - def operation( - self, - name: str, - *, - flavor: str | None = None, - **attrs: Any, - ): - return _OperationManager( - self.runtime, name, flavor=flavor, attrs=attrs - ) - - def entrypoint( - self, - name: str | None = None, - *, - flavor: str | None = None, - **attrs: Any, - ): - def decorator(func): - operation_name = name or getattr(func, "__name__", "operation") - return self.runtime.operation( - operation_name, - flavor=flavor, - **attrs, - )(func) - - return decorator - - def start_operation( - self, - name: str, - *, - flavor: str | None = None, - parent_context: Any = None, - timings: dict[str, float] | None = None, - emit_log: bool = True, - record_metric: bool = True, - **attrs: Any, - ) -> dict[str, Any]: - handle = { - "operation": None, - "operation_token": None, - "timings_token": None, - "start_token": None, - "context_token": None, - } - if not self.runtime.enabled: - return handle - try: - operation = OperationObservabilityContext( - config=self.runtime.config, - name=self.runtime._safe_str(name), - flavor=flavor, - attributes={ - key: value - for key, value in attrs.items() - if value is not None - }, - timings_ms={}, - emit_log=emit_log, - record_metric=record_metric, - ) - operation.context_token = _state._OPERATION_CONTEXT.set(operation) - handle["operation"] = operation - handle["operation_token"] = operation.context_token - if timings is not None: - handle["timings_token"] = _state._TIMINGS.set(timings) - handle["start_token"] = _state._TURN_START.set(time.perf_counter()) - if parent_context is not None and self.runtime.tracer is not None: - try: - from opentelemetry import context as otel_context - - handle["context_token"] = otel_context.attach( - parent_context - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "operation.context_attach", - exc, - ) - if self.runtime.tracer is not None: - operation.span_handle = self.runtime._start_span( - self.runtime._span_name(operation.name), - operation.span_attributes(), - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "operation.start", exc, name=name - ) - return handle - - def end_operation( - self, - handle: dict[str, Any] | None, - error: BaseException | None = None, - ) -> None: - if not handle: - return - operation = handle.get("operation") - try: - if operation is not None and error is not None: - operation.error = ErrorRecord( - type=type(error).__name__, - message=self.runtime._safe_str(error), - handled=False, - stack=self.runtime._safe_traceback(error), - ) - self.runtime.record_error_metric( - operation.metric_attributes( - error_type=type(error).__name__ - ) - ) - if operation is not None: - self.runtime.complete_operation(operation) - if operation is not None: - self.runtime._end_span(operation.span_handle, error) - except BaseException as exc: - self.runtime.log_observability_failure("operation.end", exc) - finally: - context_token = handle.get("context_token") - if context_token is not None: - try: - from opentelemetry import context as otel_context - - otel_context.detach(context_token) - except BaseException as exc: - self.runtime.log_observability_failure( - "operation.context_detach", - exc, - ) - for var, key in ( - (_state._TIMINGS, "timings_token"), - (_state._TURN_START, "start_token"), - (_state._OPERATION_CONTEXT, "operation_token"), - ): - token = handle.get(key) - if token is not None: - try: - var.reset(token) - except BaseException as exc: - self.runtime.log_observability_failure( - "operation.context_reset", - exc, - token=key, - ) - - def complete_operation( - self, - operation: OperationObservabilityContext, - ) -> None: - if operation.metric_recorded: - return - operation.metric_recorded = True - if operation.record_metric: - self.runtime.record_operation_metric( - operation.duration_seconds(), - operation.metric_attributes(), - ) - if operation.emit_log: - self.runtime.emit_operation_log(operation) - - @contextmanager - def collect_timings(self, name: str = "operation", **attrs: Any): - timings: dict[str, float] = {} - handle = self.runtime.start_scope(timings, name=name, **attrs) - error: BaseException | None = None - try: - yield timings - except BaseException as exc: - error = exc - raise - finally: - self.runtime.end_scope(handle, error) - - def start_scope( - self, - timings: dict[str, float], - *, - name: str = "operation", - parent_context: Any = None, - **attrs: Any, - ) -> dict[str, Any]: - if self.runtime.current_operation() is None: - return { - "operation_handle": self.runtime.start_operation( - name, - parent_context=parent_context, - timings=timings, - **attrs, - ) - } - handle = { - "operation_handle": None, - "timings_token": None, - "start_token": None, - "context_token": None, - "span": None, - } - try: - handle["timings_token"] = _state._TIMINGS.set(timings) - except BaseException as exc: - self.runtime.log_observability_failure("scope.timings_set", exc) - try: - handle["start_token"] = _state._TURN_START.set(time.perf_counter()) - except BaseException as exc: - self.runtime.log_observability_failure("scope.start_set", exc) - if parent_context is not None and self.runtime.tracer is not None: - try: - from opentelemetry import context as otel_context - - handle["context_token"] = otel_context.attach(parent_context) - except BaseException as exc: - self.runtime.log_observability_failure( - "scope.context_attach", exc - ) - try: - if self.runtime.tracer is not None: - handle["span"] = self.runtime._start_span(name, attrs) - except BaseException as exc: - self.runtime.log_observability_failure( - "scope.span_start", exc, span=name - ) - handle["span"] = None - return handle - - def annotate( - self, - handle: dict[str, Any] | None = None, - **attrs: Any, - ) -> None: - try: - if handle: - span_handle = handle.get("span") - if span_handle is not None: - _cm, span = span_handle - for key, value in attrs.items(): - if value is not None: - span.set_attribute(key, value) - context = self.runtime.current_context() - if context is not None: - for key, value in attrs.items(): - context.set_attribute(key, value) - operation = self.runtime.current_operation() - if operation is not None: - for key, value in attrs.items(): - operation.set_attribute(key, value) - self.runtime._set_current_span_attributes( - operation.span_attributes() - ) - except BaseException as exc: - self.runtime.log_observability_failure("scope.annotate", exc) - - def end_scope( - self, - handle: dict[str, Any] | None, - error: BaseException | None = None, - ) -> None: - if not handle: - return - operation_handle = handle.get("operation_handle") - if operation_handle is not None: - self.runtime.end_operation(operation_handle, error) - return - try: - self.runtime._end_span(handle.get("span"), error) - except BaseException as exc: - self.runtime.log_observability_failure("scope.span_end", exc) - context_token = handle.get("context_token") - if context_token is not None: - try: - from opentelemetry import context as otel_context - - otel_context.detach(context_token) - except BaseException as exc: - self.runtime.log_observability_failure( - "scope.context_detach", exc - ) - for var, key in ( - (_state._TIMINGS, "timings_token"), - (_state._TURN_START, "start_token"), - ): - token = handle.get(key) - if token is not None: - try: - var.reset(token) - except BaseException as exc: - self.runtime.log_observability_failure( - "scope.context_reset", - exc, - token=key, - ) - - def mark(self, key: str, ms: float) -> None: - try: - timings = _state._TIMINGS.get() - if timings is not None: - timings[key] = round(float(ms), 1) - except BaseException as exc: - self.runtime.log_observability_failure("scope.mark", exc, key=key) - - def mark_ttft(self, key: str = "ttft_ms") -> None: - try: - start = _state._TURN_START.get() - if start is not None: - self.runtime.mark(key, (time.perf_counter() - start) * 1000.0) - except BaseException as exc: - self.runtime.log_observability_failure("scope.mark_ttft", exc) - - def mark_ttft_attribute(self, key: str = "ttft_ms") -> None: - try: - start = _state._TURN_START.get() - if start is None: - return - self.runtime.annotate( - **{key: round((time.perf_counter() - start) * 1000.0, 1)} - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "scope.mark_ttft_attribute", exc - ) - - def _start_implicit_operation( - self, - segment_name: str, - attrs: dict[str, Any], - ) -> dict[str, Any] | None: - if ( - self.runtime.current_operation() is not None - or self.runtime.current_context() is not None - ): - return None - operation_name = attrs.get("operation") or segment_name - flavor = attrs.get("flavor") - operation_attrs = { - key: value - for key, value in attrs.items() - if key not in {"operation", "flavor"} and value is not None - } - return self.runtime.start_operation( - self.runtime._safe_str(operation_name), - flavor=self.runtime._safe_str(flavor) - if flavor is not None - else None, - **operation_attrs, - ) - - -class _OperationManager: - def __init__( - self, - runtime: ObservabilityRuntime, - name: str, - *, - flavor: str | None, - attrs: dict[str, Any], - ) -> None: - self.runtime = runtime - self.name = name - self.flavor = flavor - self.attrs = attrs - self.handle: dict[str, Any] | None = None - - def __enter__(self): - self.handle = self.runtime.start_operation( - self.name, - flavor=self.flavor, - **self.attrs, - ) - return self.runtime.current_operation() - - def __exit__(self, exc_type, exc, _traceback) -> bool: - self.runtime.end_operation(self.handle, exc) - return False - - async def __aenter__(self): - return self.__enter__() - - async def __aexit__(self, exc_type, exc, traceback) -> bool: - return self.__exit__(exc_type, exc, traceback) - - def __call__(self, func): - if inspect.iscoroutinefunction(func): - - @wraps(func) - async def async_wrapper(*args, **kwargs): - async with self.runtime.operation( - self.name, - flavor=self.flavor, - **self.attrs, - ): - return await func(*args, **kwargs) - - return async_wrapper - - @wraps(func) - def wrapper(*args, **kwargs): - with self.runtime.operation( - self.name, - flavor=self.flavor, - **self.attrs, - ): - return func(*args, **kwargs) - - return wrapper diff --git a/policyengine_observability/_requests.py b/policyengine_observability/_requests.py deleted file mode 100644 index b326e91..0000000 --- a/policyengine_observability/_requests.py +++ /dev/null @@ -1,291 +0,0 @@ -"""Request lifecycle, response metadata, and cleanup.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -from . import _state -from ._state import REQUEST_ID_HEADER, TRACEPARENT_HEADER -from .context import ( - OperationObservabilityContext, - RequestObservabilityContext, -) - -if TYPE_CHECKING: - from .runtime import ObservabilityRuntime - - -class RequestLifecycle: - def __init__(self, runtime: ObservabilityRuntime) -> None: - self.runtime = runtime - - def begin_request( - self, - context: RequestObservabilityContext, - *, - carrier: Any = None, - ) -> None: - if not self.runtime.enabled: - return - try: - context.context_token = _state._REQUEST_CONTEXT.set(context) - context.set_attribute("endpoint", context.endpoint) - self.runtime._begin_request_operation(context) - self.runtime._start_request_span(context, carrier=carrier) - self.runtime.record_active_request(1, context.metric_attributes()) - except BaseException as exc: - self.runtime.log_observability_failure("request.begin", exc) - - def _begin_request_operation( - self, - context: RequestObservabilityContext, - ) -> None: - try: - parent_operation = _state._OPERATION_CONTEXT.get() - timings = context.timings_ms - timing_counts = context.timing_counts - segment_tree = context.segment_tree - segment_sequence = context.segment_sequence - if context.internal_dispatch and parent_operation is not None: - timings = parent_operation.timings_ms - timing_counts = parent_operation.timing_counts - segment_tree = parent_operation.segment_tree - segment_sequence = parent_operation.segment_sequence - context.timings_ms = timings - context.timing_counts = timing_counts - context.segment_tree = segment_tree - context.segment_sequence = segment_sequence - operation = OperationObservabilityContext( - config=context.config, - name=context.route, - flavor="http", - attributes={ - "route": context.route, - "method": context.method, - "endpoint": context.endpoint, - "path": context.path, - }, - timings_ms=timings, - timing_counts=timing_counts, - segment_tree=segment_tree, - segment_sequence=segment_sequence, - emit_log=False, - record_metric=False, - ) - operation.context_token = _state._OPERATION_CONTEXT.set(operation) - context.operation_context = operation - context.operation_token = operation.context_token - except BaseException as exc: - self.runtime.log_observability_failure( - "request.operation_begin", - exc, - request_id=getattr(context, "request_id", None), - ) - - def finish_request(self, status_code: int) -> dict[str, str]: - headers = self.runtime.prepare_response(status_code) - self.runtime.complete_request(status_code) - return headers - - def prepare_response(self, status_code: int) -> dict[str, str]: - if not self.runtime.enabled: - return {} - headers: dict[str, str] = {} - try: - context = self.runtime.current_context() - if context is None: - return headers - context.status_code = status_code - self.runtime._set_current_span_attributes( - context.span_attributes() - ) - if context.operation_context is not None: - context.operation_context.set_attribute( - "status_code", - str(status_code), - ) - headers[REQUEST_ID_HEADER] = context.request_id - traceparent = self.runtime.traceparent_header() - if traceparent: - headers[TRACEPARENT_HEADER] = traceparent - if status_code == 429: - context.set_attribute("rate_limited", True) - return headers - except BaseException as exc: - self.runtime.log_observability_failure( - "request.prepare_response", exc - ) - return headers - - def complete_request(self, status_code: int | None = None) -> None: - if not self.runtime.enabled: - return - try: - context = self.runtime.current_context() - if context is None: - return - if status_code is not None: - context.status_code = status_code - self.runtime._set_current_span_attributes( - context.span_attributes() - ) - if context.request_metric_recorded: - return - context.request_metric_recorded = True - if context.status_code == 429: - self.runtime.record_rate_limited_metric( - context.metric_attributes() - ) - self.runtime.record_request_metric( - context.duration_seconds(), - context.metric_attributes(), - ) - self.runtime._close_active_request(context) - except BaseException as exc: - self.runtime.log_observability_failure("request.complete", exc) - - def update_request_route( - self, - *, - route: str | None = None, - endpoint: str | None = None, - ) -> None: - if not self.runtime.enabled: - return - try: - context = self.runtime.current_context() - if context is None: - return - route_changed = bool(route and route != context.route) - old_active_attributes = ( - context.metric_attributes() - if route_changed and not context.active_closed - else None - ) - if route: - context.route = route - if context.operation_context is not None: - context.operation_context.name = route - context.operation_context.set_attribute("route", route) - if endpoint: - context.endpoint = endpoint - context.set_attribute("endpoint", endpoint) - if context.operation_context is not None: - context.operation_context.set_attribute( - "endpoint", - endpoint, - ) - self.runtime._set_current_span_attributes( - context.span_attributes() - ) - span = context.server_span - update_name = getattr(span, "update_name", None) - if route and update_name is not None: - update_name(route) - if old_active_attributes is not None: - self.runtime.record_active_request(-1, old_active_attributes) - self.runtime.record_active_request( - 1, context.metric_attributes() - ) - except BaseException as exc: - self.runtime.log_observability_failure("request.update_route", exc) - - def teardown_request(self, exc: BaseException | None = None) -> None: - if not self.runtime.enabled: - return - context = self.runtime.current_context() - if context is None: - return - try: - if exc is not None: - self.runtime.record_error( - exc, - handled=False, - status_code=context.status_code or 500, - ) - self.runtime._close_active_request(context) - self.runtime.emit_request_log(context) - except BaseException as observability_exc: - self.runtime.log_observability_failure( - "request.teardown", - observability_exc, - ) - finally: - self.runtime._close_request_span(context, exc) - self.runtime._reset_request_operation_context(context) - self.runtime._reset_request_context(context) - - def set_attribute(self, key: str, value: Any) -> None: - if not self.runtime.enabled: - return - try: - context = self.runtime.current_context() - if context is not None: - context.set_attribute(key, value) - if context.operation_context is not None: - context.operation_context.set_attribute(key, value) - self.runtime._set_current_span_attributes( - context.span_attributes(**{f"policyengine.{key}": value}) - ) - operation = self.runtime.current_operation() - if operation is not None and operation is not getattr( - context, "operation_context", None - ): - operation.set_attribute(key, value) - self.runtime._set_current_span_attributes( - operation.span_attributes(**{f"policyengine.{key}": value}) - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "request.set_attribute", - exc, - attribute=key, - ) - - def _close_active_request( - self, - context: RequestObservabilityContext, - ) -> None: - try: - if context.active_closed: - return - context.active_closed = True - self.runtime.record_active_request(-1, context.metric_attributes()) - except BaseException as exc: - self.runtime.log_observability_failure( - "request.close_active", - exc, - request_id=getattr(context, "request_id", None), - ) - - def _reset_request_operation_context( - self, - context: RequestObservabilityContext, - ) -> None: - token = context.operation_token - if token is None: - return - try: - _state._OPERATION_CONTEXT.reset(token) - except BaseException as exc: - self.runtime.log_observability_failure( - "request.operation_context_reset", - exc, - request_id=getattr(context, "request_id", None), - ) - - def _reset_request_context( - self, - context: RequestObservabilityContext, - ) -> None: - token = context.context_token - if token is None: - return - try: - _state._REQUEST_CONTEXT.reset(token) - except BaseException as exc: - self.runtime.log_observability_failure( - "request.context_reset", - exc, - request_id=getattr(context, "request_id", None), - ) diff --git a/policyengine_observability/_state.py b/policyengine_observability/_state.py deleted file mode 100644 index a601d3b..0000000 --- a/policyengine_observability/_state.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Shared context variables for requests, operations, and segments.""" - -from __future__ import annotations - -from contextvars import ContextVar -from typing import TYPE_CHECKING - -from .context import ( - OperationObservabilityContext, - RequestObservabilityContext, - SegmentTimingNode, -) - -if TYPE_CHECKING: - from .runtime import ObservabilityRuntime - -OBSERVABILITY_INTERNAL_DISPATCH_HEADER = "X-PolicyEngine-Internal-Dispatch" -REQUEST_ID_HEADER = "X-PolicyEngine-Request-Id" -TRACEPARENT_HEADER = "traceparent" - -_REQUEST_CONTEXT: ContextVar[RequestObservabilityContext | None] = ContextVar( - "policyengine_request_observability_context", - default=None, -) -_OPERATION_CONTEXT: ContextVar[OperationObservabilityContext | None] = ( - ContextVar( - "policyengine_operation_observability_context", - default=None, - ) -) -_TIMINGS: ContextVar[dict[str, float] | None] = ContextVar( - "policyengine_observability_timings", - default=None, -) -_TURN_START: ContextVar[float | None] = ContextVar( - "policyengine_observability_turn_start", - default=None, -) -_SEGMENT_STACK: ContextVar[tuple[tuple[int, SegmentTimingNode], ...]] = ( - ContextVar( - "policyengine_observability_segment_stack", - default=(), - ) -) - - -class ContextState: - def __init__(self, runtime: ObservabilityRuntime) -> None: - self.runtime = runtime - - def current_context(self) -> RequestObservabilityContext | None: - try: - return _REQUEST_CONTEXT.get() - except BaseException as exc: - self.runtime.log_observability_failure("context.current", exc) - return None - - def current_operation( - self, - ) -> OperationObservabilityContext | None: - try: - return _OPERATION_CONTEXT.get() - except BaseException as exc: - self.runtime.log_observability_failure("operation.current", exc) - return None diff --git a/policyengine_observability/_tracing.py b/policyengine_observability/_tracing.py deleted file mode 100644 index b8dfc6c..0000000 --- a/policyengine_observability/_tracing.py +++ /dev/null @@ -1,434 +0,0 @@ -"""Trace initialization, propagation, and span lifecycle.""" - -from __future__ import annotations - -from collections.abc import Iterator -from contextlib import contextmanager -from typing import TYPE_CHECKING, Any - -from ._state import TRACEPARENT_HEADER -from .context import ( - RequestObservabilityContext, -) - -if TYPE_CHECKING: - from .runtime import ObservabilityRuntime - - -def _is_safe_span_value(value: Any) -> bool: - return isinstance(value, str | bool | int | float) - - -class TraceRecorder: - def __init__(self, runtime: ObservabilityRuntime) -> None: - self.runtime = runtime - - def traceparent_header(self) -> str | None: - if not self.runtime.enabled or self.runtime.propagate is None: - return None - try: - carrier: dict[str, str] = {} - self.runtime.propagate.inject(carrier) - return carrier.get(TRACEPARENT_HEADER) - except BaseException as exc: - self.runtime.log_observability_failure( - "request.traceparent_header", exc - ) - return None - - def capture_context(self): - if self.runtime.tracer is None: - return None - try: - from opentelemetry import context as otel_context - - return otel_context.get_current() - except BaseException as exc: - self.runtime.log_observability_failure("otel.capture_context", exc) - return None - - def instrument_fastapi(self, app: Any) -> None: - if not self.runtime.enabled or not self.runtime.config.otel_enabled: - return - try: - from opentelemetry.instrumentation.fastapi import ( - FastAPIInstrumentor, - ) - - FastAPIInstrumentor.instrument_app(app) - except BaseException as exc: - self.runtime.log_observability_failure( - "fastapi.auto_instrument", - exc, - ) - - def instrument_httpx(self) -> None: - if ( - not self.runtime.enabled - or not self.runtime.config.otel_enabled - or self.runtime._httpx_instrumented - ): - return - try: - from opentelemetry.instrumentation.httpx import ( - HTTPXClientInstrumentor, - ) - - HTTPXClientInstrumentor().instrument() - self.runtime._httpx_instrumented = True - except BaseException as exc: - self.runtime.log_observability_failure( - "httpx.auto_instrument", exc - ) - - def _configure_otel(self) -> None: - try: - from opentelemetry import metrics, propagate, trace - from opentelemetry.sdk.metrics import MeterProvider - from opentelemetry.sdk.resources import ( - DEPLOYMENT_ENVIRONMENT, - SERVICE_NAME, - Resource, - ) - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.trace import SpanKind, Status, StatusCode - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.configure_imports", exc - ) - return - - try: - resource = Resource.create( - { - SERVICE_NAME: self.runtime.config.service_name, - DEPLOYMENT_ENVIRONMENT: self.runtime.config.environment, - "service.role": self.runtime.config.service_role, - } - ) - tracer_provider = TracerProvider(resource=resource) - metric_readers = [] - if self.runtime.config.otlp_endpoint: - self.runtime._add_trace_exporter(tracer_provider) - metric_reader = self.runtime._metric_reader() - if metric_reader is not None: - metric_readers.append(metric_reader) - self.runtime.tracer_provider = tracer_provider - try: - trace.set_tracer_provider(tracer_provider) - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.set_tracer_provider", - exc, - ) - try: - self.runtime.meter_provider = MeterProvider( - resource=resource, - metric_readers=metric_readers, - ) - metrics.set_meter_provider(self.runtime.meter_provider) - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.set_meter_provider", - exc, - ) - self.runtime.trace = trace - self.runtime.propagate = propagate - self.runtime.SpanKind = SpanKind - self.runtime.Status = Status - self.runtime.StatusCode = StatusCode - tracer_name = ( - self.runtime.config.tracer_name - or self.runtime.config.service_name - ) - meter_name = ( - self.runtime.config.meter_name - or self.runtime.config.service_name - ) - self.runtime.tracer = trace.get_tracer(tracer_name) - self.runtime.meter = metrics.get_meter(meter_name) - self.runtime._configure_instruments() - except BaseException as exc: - self.runtime.log_observability_failure("otel.configure", exc) - - def _add_trace_exporter(self, tracer_provider) -> None: - try: - if self.runtime.config.otlp_protocol.startswith("http"): - from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( - OTLPSpanExporter, - ) - else: - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( - OTLPSpanExporter, - ) - from opentelemetry.sdk.trace.export import BatchSpanProcessor - - tracer_provider.add_span_processor( - BatchSpanProcessor(OTLPSpanExporter()) - ) - except BaseException as exc: - self.runtime.log_observability_failure("otel.trace_exporter", exc) - - def _metric_reader(self): - try: - if self.runtime.config.otlp_protocol.startswith("http"): - from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( - OTLPMetricExporter, - ) - else: - from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( - OTLPMetricExporter, - ) - from opentelemetry.sdk.metrics.export import ( - PeriodicExportingMetricReader, - ) - - return PeriodicExportingMetricReader(OTLPMetricExporter()) - except BaseException as exc: - self.runtime.log_observability_failure("otel.metric_exporter", exc) - return None - - def _start_request_span( - self, - context: RequestObservabilityContext, - *, - carrier: Any = None, - ) -> None: - if self.runtime.tracer is None: - return - attrs = context.span_attributes() - parent_context = self.runtime._extract_context(carrier) - try: - context.server_span_cm = self.runtime.tracer.start_as_current_span( - context.route, - context=parent_context, - kind=self.runtime.SpanKind.SERVER - if self.runtime.SpanKind - else None, - attributes=attrs, - ) - context.server_span = context.server_span_cm.__enter__() - except BaseException as exc: - context.server_span_cm = None - context.server_span = None - self.runtime.log_observability_failure( - "otel.request_span_enter", exc - ) - - def _close_request_span( - self, - context: RequestObservabilityContext, - exc: BaseException | None, - ) -> None: - if context.span_closed: - return - context.span_closed = True - span_cm = context.server_span_cm - if span_cm is None: - return - try: - if exc is None: - span_cm.__exit__(None, None, None) - else: - span_cm.__exit__(type(exc), exc, exc.__traceback__) - except BaseException as observability_exc: - self.runtime.log_observability_failure( - "otel.request_span_exit", - observability_exc, - request_id=context.request_id, - ) - - @contextmanager - def _safe_span(self, name: str, attrs: dict[str, Any]) -> Iterator[Any]: - if self.runtime.tracer is None: - yield None - return - span_handle = self.runtime._start_span(name, attrs) - if span_handle is None: - yield None - return - _cm, span = span_handle - try: - yield span - except BaseException as exc: - try: - self.runtime._end_span(span_handle, exc) - except BaseException as observability_exc: - self.runtime.log_observability_failure( - "otel.span_exit", - observability_exc, - span=name, - ) - raise - else: - try: - self.runtime._end_span(span_handle) - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.span_exit", - exc, - span=name, - ) - - def _start_span(self, name: str, attrs: dict[str, Any]): - try: - span_cm = self.runtime.tracer.start_as_current_span(name) - span = span_cm.__enter__() - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.span_enter", exc, span=name - ) - return None - try: - for key, value in attrs.items(): - if value is not None: - span.set_attribute(key, value) - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.span_attributes", - exc, - span=name, - ) - return span_cm, span - - def _end_span( - self, - span_handle, - error: BaseException | None = None, - ) -> None: - if span_handle is None: - return - span_cm, span = span_handle - try: - if error is not None: - self.runtime._record_exception_on_span( - span, - error, - handled=False, - status_code=500, - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.span_error_status", exc - ) - try: - span_cm.__exit__(None, None, None) - except BaseException as exc: - self.runtime.log_observability_failure("otel.span_exit", exc) - - def _segment_span_attributes( - self, - attrs: dict[str, Any], - ) -> dict[str, Any]: - context = self.runtime.current_context() - operation = self.runtime.current_operation() - span_attrs = { - key: value for key, value in attrs.items() if value is not None - } - if context is not None: - span_attrs = {**context.span_attributes(), **span_attrs} - elif operation is not None: - span_attrs = {**operation.span_attributes(), **span_attrs} - return span_attrs - - def _span_name(self, segment_name: str) -> str: - if not self.runtime.config.span_prefix: - return segment_name - return f"{self.runtime.config.span_prefix}.{segment_name}" - - def _set_current_span_attributes(self, attrs: dict[str, Any]) -> None: - span = self.runtime._current_span() - if span is None: - return - try: - for key, value in attrs.items(): - if value is not None: - span.set_attribute(key, value) - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.set_span_attributes", exc - ) - - def _current_span(self): - if self.runtime.trace is None: - return None - try: - return self.runtime.trace.get_current_span() - except BaseException as exc: - self.runtime.log_observability_failure("otel.current_span", exc) - return None - - def _trace_ids(self) -> tuple[str | None, str | None]: - span = self.runtime._current_span() - if span is None: - return None, None - try: - context = span.get_span_context() - except BaseException as exc: - self.runtime.log_observability_failure("otel.span_context", exc) - return None, None - if not getattr(context, "is_valid", False): - return None, None - return f"{context.trace_id:032x}", f"{context.span_id:016x}" - - def _extract_context(self, carrier: Any): - if self.runtime.propagate is None or carrier is None: - return None - try: - return self.runtime.propagate.extract(carrier) - except BaseException as exc: - self.runtime.log_observability_failure("otel.extract_context", exc) - return None - - def _record_exception_on_span( - self, - span, - exc: BaseException, - *, - handled: bool, - status_code: int | None, - ) -> None: - try: - span.record_exception(exc) - span.set_attribute("error.type", type(exc).__name__) - span.set_attribute("error.handled", handled) - if ( - self.runtime.Status is not None - and self.runtime.StatusCode is not None - and ( - not handled - or (status_code is not None and status_code >= 500) - ) - ): - span.set_status( - self.runtime.Status( - self.runtime.StatusCode.ERROR, - self.runtime._safe_str(exc), - ) - ) - except BaseException as observability_exc: - self.runtime.log_observability_failure( - "otel.record_exception", - observability_exc, - original_error_type=type(exc).__name__, - ) - - def _add_span_event(self, event: str, fields: dict[str, Any]) -> None: - span = self.runtime._current_span() - if span is None: - return - try: - span.add_event( - event, - { - key: value - for key, value in fields.items() - if _is_safe_span_value(value) - }, - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "otel.add_event", - exc, - event_name=event, - ) diff --git a/policyengine_observability/adapters/__init__.py b/policyengine_observability/adapters/__init__.py index 99cc2ad..96a7008 100644 --- a/policyengine_observability/adapters/__init__.py +++ b/policyengine_observability/adapters/__init__.py @@ -1 +1,6 @@ """Framework adapters for policyengine-observability.""" + +from .fastapi import instrument_fastapi +from .flask import instrument_flask + +__all__ = ["instrument_fastapi", "instrument_flask"] diff --git a/policyengine_observability/adapters/fastapi.py b/policyengine_observability/adapters/fastapi.py index 91fc0f2..7cb18da 100644 --- a/policyengine_observability/adapters/fastapi.py +++ b/policyengine_observability/adapters/fastapi.py @@ -1,302 +1,142 @@ from __future__ import annotations -import uuid +from collections.abc import Awaitable, Callable from typing import Any -from urllib.parse import parse_qs -from ..config import ObservabilityConfig -from ..context import RequestObservabilityContext -from ..runtime import ( - REQUEST_ID_HEADER, - TRACEPARENT_HEADER, - ObservabilityRuntime, - set_observability_runtime, -) +from ..runtime import ObservabilityRuntime -UNMATCHED_ROUTE = "" +_STATE_KEY = "policyengine_observability" -class FastAPIObservabilityAdapter: - def __init__( - self, - runtime: ObservabilityRuntime, - *, - static_attributes: dict[str, Any] | None = None, - ) -> None: - self.runtime = runtime - self.static_attributes = { - key: value - for key, value in (static_attributes or {}).items() - if value is not None - } +def instrument_fastapi( + app: Any, runtime: ObservabilityRuntime +) -> ObservabilityRuntime: + """Install one request lifecycle integration on a FastAPI application.""" - def instrument_app(self, app: Any) -> None: - if not self.runtime.enabled: - return - if getattr(app.state, "policyengine_observability_adapter", None): - return - app.state.policyengine_observability_adapter = self - if self.runtime.config.instrument_fastapi: - self.runtime.instrument_fastapi(app) - try: - app.add_middleware( - FastAPIObservabilityMiddleware, - adapter=self, - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "fastapi.middleware_install", - exc, - ) + existing = getattr(app.state, _STATE_KEY, None) + if isinstance(existing, ObservabilityRuntime): + return existing + try: + app.add_middleware(_ObservabilityMiddleware, runtime=runtime) + setattr(app.state, _STATE_KEY, runtime) + except Exception as exc: + runtime.diagnostics.report("fastapi.middleware_install", exc) + return runtime - def start_request(self, scope: dict[str, Any]) -> None: - try: - headers = _headers_from_scope(scope) - path = scope.get("path") or "" - route = _route_from_scope(scope) or UNMATCHED_ROUTE - endpoint = _endpoint_from_scope(scope) - request_id = headers.get(REQUEST_ID_HEADER.lower()) or str( - uuid.uuid4() - ) - context = RequestObservabilityContext( - config=self.runtime.config, - request_id=request_id, - method=scope.get("method") or "", - route=route, - path=path, - endpoint=endpoint, - query_keys=_query_keys(scope), - content_length_bytes=_int_header( - headers.get("content-length") - ), - inbound=self._inbound_metadata(scope, headers), - ) - self.runtime.begin_request(context, carrier=headers) - if self.static_attributes: - self.runtime.annotate(**self.static_attributes) - except BaseException as exc: - self.runtime.log_observability_failure( - "fastapi.before_request", - exc, - ) - def update_resolved_route(self, scope: dict[str, Any]) -> None: - route = _route_from_scope(scope) - endpoint = _endpoint_from_scope(scope) - if route or endpoint: - self.runtime.update_request_route(route=route, endpoint=endpoint) +class _ObservabilityMiddleware: + def __init__(self, app: Any, *, runtime: ObservabilityRuntime) -> None: + self.app = app + self.runtime = runtime - def _inbound_metadata( + async def __call__( self, scope: dict[str, Any], - headers: dict[str, str], - ) -> dict[str, Any]: - forwarded_for = _split_forwarded_for(headers.get("x-forwarded-for")) - x_real_ip = headers.get("x-real-ip") - client = scope.get("client") or () - remote_addr = client[0] if client else None - client_ip = None - ip_source = None - if forwarded_for: - client_ip = forwarded_for[0] - ip_source = "x_forwarded_for" - elif x_real_ip: - client_ip = x_real_ip - ip_source = "x_real_ip" - elif remote_addr: - client_ip = remote_addr - ip_source = "remote_addr" - metadata = { - "ip_source": ip_source, - "user_agent": headers.get("user-agent"), - "origin": headers.get("origin"), - "referer": headers.get("referer"), - "host": headers.get("host"), - "content_length_bytes": _int_header(headers.get("content-length")), - } - if self.runtime.config.log_raw_ip: - metadata["client_ip"] = client_ip - metadata["forwarded_for"] = forwarded_for - metadata["x_real_ip"] = x_real_ip - return metadata - - -class FastAPIObservabilityMiddleware: - def __init__( - self, - app: Any, - *, - adapter: FastAPIObservabilityAdapter, + receive: Callable[[], Awaitable[dict[str, Any]]], + send: Callable[[dict[str, Any]], Awaitable[None]], ) -> None: - self.app = app - self.adapter = adapter - - async def __call__(self, scope, receive, send) -> None: if scope.get("type") != "http": await self.app(scope, receive, send) return - self.adapter.start_request(scope) status_code: int | None = None + error: Exception | None = None completed = False + try: + self.runtime.begin_request( + headers=_headers_from_scope(scope), + method=str(scope.get("method") or ""), + route=_route_from_scope(scope) or "", + ) + except Exception as exc: + self.runtime.diagnostics.report("fastapi.request_begin", exc) - async def send_wrapper(message) -> None: - nonlocal completed - nonlocal status_code - - if message["type"] == "http.response.start": + async def send_observed(message: dict[str, Any]) -> None: + nonlocal completed, status_code + if message.get("type") == "http.response.start": status_code = int(message.get("status") or 0) - self.adapter.update_resolved_route(scope) - response_headers = self.adapter.runtime.prepare_response( - status_code - ) - if response_headers: + self._update_route(scope) + try: message = { **message, - "headers": _merge_response_headers( - message.get("headers") or [], - response_headers, + "headers": _merge_headers( + list(message.get("headers") or []), + self.runtime.response_headers(), ), } - await send(message) - return - - if message["type"] == "http.response.body" and not message.get( + except Exception as exc: + self.runtime.diagnostics.report( + "fastapi.response_headers", exc + ) + await send(message) + if message.get("type") == "http.response.body" and not message.get( "more_body", False ): - try: - await send(message) - finally: - completed = True - self.adapter.update_resolved_route(scope) - self.adapter.runtime.complete_request(status_code) - self.adapter.runtime.teardown_request(None) - return - - await send(message) + self._update_route(scope) + self._end(status_code=status_code) + completed = True try: - await self.app(scope, receive, send_wrapper) - except BaseException as exc: - if not completed: - error_status = status_code or 500 - self.adapter.update_resolved_route(scope) - self.adapter.runtime.prepare_response(error_status) - self.adapter.runtime.complete_request(error_status) - self.adapter.runtime.teardown_request(exc) - completed = True + await self.app(scope, receive, send_observed) + except Exception as exc: + error = exc raise finally: if not completed: - self.adapter.update_resolved_route(scope) - self.adapter.runtime.complete_request(status_code) - self.adapter.runtime.teardown_request(None) + self._update_route(scope) + self._end( + status_code=status_code or (500 if error else None), + error=error, + ) + def _update_route(self, scope: dict[str, Any]) -> None: + try: + route = _route_from_scope(scope) + if route: + self.runtime.update_request_route(route) + except Exception as exc: + self.runtime.diagnostics.report("fastapi.route_update", exc) -def init_fastapi_observability( - app: Any, - *, - config: ObservabilityConfig | None = None, - runtime: ObservabilityRuntime | None = None, - service_name: str, - service_role: str = "api", - span_prefix: str | None = None, - segment_registry=None, - static_attributes: dict[str, Any] | None = None, -) -> ObservabilityRuntime: - existing = getattr(app.state, "policyengine_observability", None) - if existing: - return existing - runtime = runtime or ObservabilityRuntime( - config - or ObservabilityConfig.from_env( - service_name=service_name, - service_role=service_role, - span_prefix=span_prefix, - ), - segment_registry=segment_registry, - ) - runtime.configure() - app.state.policyengine_observability = runtime - set_observability_runtime(runtime) - FastAPIObservabilityAdapter( - runtime, - static_attributes=static_attributes, - ).instrument_app(app) - return runtime + def _end( + self, + *, + status_code: int | None, + error: BaseException | None = None, + ) -> None: + try: + self.runtime.end_request(status_code=status_code, error=error) + except Exception as exc: + self.runtime.diagnostics.report("fastapi.request_finish", exc) def _headers_from_scope(scope: dict[str, Any]) -> dict[str, str]: headers: dict[str, str] = {} - for key, value in scope.get("headers") or []: + for raw_key, raw_value in scope.get("headers") or []: try: - header_key = key.decode("latin-1").lower() - header_value = value.decode("latin-1") - except BaseException: + key = raw_key.decode("latin-1").lower() + value = raw_value.decode("latin-1") + except (AttributeError, UnicodeDecodeError): continue - if header_key in headers: - headers[header_key] = f"{headers[header_key]},{header_value}" - else: - headers[header_key] = header_value + headers[key] = f"{headers[key]},{value}" if key in headers else value return headers def _route_from_scope(scope: dict[str, Any]) -> str | None: - route = scope.get("route") - route_path = getattr(route, "path", None) - if route_path: - return str(route_path) - return None - - -def _endpoint_from_scope(scope: dict[str, Any]) -> str | None: - endpoint = scope.get("endpoint") - if endpoint is None: - return None - endpoint_name = getattr(endpoint, "__name__", None) - return endpoint_name or str(endpoint) - - -def _query_keys(scope: dict[str, Any]) -> list[str]: - query_string = scope.get("query_string") or b"" - try: - decoded = query_string.decode("latin-1") - except AttributeError: - decoded = str(query_string) - return sorted(parse_qs(decoded, keep_blank_values=True).keys()) + path = getattr(scope.get("route"), "path", None) + return str(path) if path else None -def _merge_response_headers( - existing_headers: list[tuple[bytes, bytes]], - headers: dict[str, str], +def _merge_headers( + existing: list[tuple[bytes, bytes]], values: dict[str, str] ) -> list[tuple[bytes, bytes]]: - response_header_names = {key.lower().encode("latin-1") for key in headers} + replacements = {key.lower().encode("latin-1") for key in values} merged = [ (key, value) - for key, value in existing_headers - if key.lower() not in response_header_names + for key, value in existing + if key.lower() not in replacements ] merged.extend( - ( - key.encode("latin-1"), - value.encode("latin-1"), - ) - for key, value in headers.items() - if key in {REQUEST_ID_HEADER, TRACEPARENT_HEADER} + (key.encode("latin-1"), value.encode("latin-1")) + for key, value in values.items() ) return merged - - -def _split_forwarded_for(value: str | None) -> list[str]: - if not value: - return [] - return [part.strip() for part in value.split(",") if part.strip()] - - -def _int_header(value: str | None) -> int | None: - if value is None: - return None - try: - return int(value) - except ValueError: - return None diff --git a/policyengine_observability/adapters/flask.py b/policyengine_observability/adapters/flask.py index 5786392..1bb0ef5 100644 --- a/policyengine_observability/adapters/flask.py +++ b/policyengine_observability/adapters/flask.py @@ -1,132 +1,147 @@ from __future__ import annotations -import uuid from typing import Any -from ..config import ObservabilityConfig -from ..context import RequestObservabilityContext -from ..runtime import ( - OBSERVABILITY_INTERNAL_DISPATCH_HEADER, - REQUEST_ID_HEADER, - ObservabilityRuntime, - set_observability_runtime, -) - - -class FlaskObservabilityAdapter: - def __init__(self, runtime: ObservabilityRuntime) -> None: - self.runtime = runtime +from ..runtime import ObservabilityRuntime - def instrument_app(self, app: Any) -> None: - if not self.runtime.enabled: - return - if app.extensions.get("policyengine_observability_adapter"): - return - app.extensions["policyengine_observability_adapter"] = self +_EXTENSION_KEY = "policyengine_observability" +_CALLBACK_REGISTRIES = ( + "before_request_funcs", + "after_request_funcs", + "teardown_request_funcs", +) +_MISSING = object() - @app.before_request - def _start_observed_request() -> None: - self.start_request() - @app.after_request - def _finish_observed_request(response): - headers = self.runtime.finish_request(response.status_code) - for key, value in headers.items(): - response.headers[key] = value - return response +def instrument_flask( + app: Any, runtime: ObservabilityRuntime +) -> ObservabilityRuntime: + """Install one request lifecycle integration on a Flask application.""" - @app.teardown_request - def _emit_observed_request(exc) -> None: - self.runtime.teardown_request(exc) + try: + extensions = app.extensions + existing = extensions.get(_EXTENSION_KEY, _MISSING) + except Exception as exc: + runtime.diagnostics.report("flask.callback_install", exc) + return runtime + if isinstance(existing, ObservabilityRuntime): + return existing - def start_request(self) -> None: + def _begin_observed_request() -> None: try: from flask import request - route = request.url_rule.rule if request.url_rule else request.path - request_id = request.headers.get(REQUEST_ID_HEADER) or str( - uuid.uuid4() + route = ( + request.url_rule.rule if request.url_rule else "" ) - context = RequestObservabilityContext( - config=self.runtime.config, - request_id=request_id, + runtime.begin_request( + headers=dict(request.headers), method=request.method, route=route, - path=request.path, - endpoint=request.endpoint, - query_keys=sorted(request.args.keys()), - content_length_bytes=request.content_length, - inbound=self._inbound_metadata(request), - internal_dispatch=( - request.headers.get(OBSERVABILITY_INTERNAL_DISPATCH_HEADER) - == "1" - ), ) - self.runtime.begin_request(context, carrier=request.headers) - except BaseException as exc: - self.runtime.log_observability_failure("flask.before_request", exc) + except Exception as exc: + runtime.diagnostics.report("flask.request_begin", exc) + + def _finish_observed_request(response: Any) -> Any: + try: + from flask import request + + if request.url_rule is not None: + runtime.update_request_route(request.url_rule.rule) + except Exception as exc: + runtime.diagnostics.report("flask.request_route", exc) + try: + runtime.update_request_status(response.status_code) + except Exception as exc: + runtime.diagnostics.report("flask.request_status", exc) + try: + for key, value in runtime.response_headers().items(): + response.headers[key] = value + except Exception as exc: + runtime.diagnostics.report("flask.response_headers", exc) + return response + + def _close_observed_request(error: BaseException | None) -> None: + try: + runtime.end_request( + status_code=500 if error is not None else None, + error=error, + ) + except Exception as exc: + runtime.diagnostics.report("flask.request_teardown", exc) - def _inbound_metadata(self, request) -> dict: - forwarded_for = _split_forwarded_for( - request.headers.get("X-Forwarded-For") + snapshots: dict[str, dict[Any, list[Any]]] | None = None + try: + snapshots = _snapshot_callback_registries(app) + app.before_request(_begin_observed_request) + _move_callback_to_start( + app.before_request_funcs, + _begin_observed_request, ) - x_real_ip = request.headers.get("X-Real-IP") - remote_addr = request.remote_addr - client_ip = None - ip_source = None - if forwarded_for: - client_ip = forwarded_for[0] - ip_source = "x_forwarded_for" - elif x_real_ip: - client_ip = x_real_ip - ip_source = "x_real_ip" - elif remote_addr: - client_ip = remote_addr - ip_source = "remote_addr" - metadata = { - "ip_source": ip_source, - "user_agent": request.headers.get("User-Agent"), - "origin": request.headers.get("Origin"), - "referer": request.headers.get("Referer"), - "host": request.host, - "content_length_bytes": request.content_length, - } - if self.runtime.config.log_raw_ip: - metadata["client_ip"] = client_ip - metadata["forwarded_for"] = forwarded_for - metadata["x_real_ip"] = x_real_ip - return metadata + app.after_request(_finish_observed_request) + _move_callback_to_start( + app.after_request_funcs, + _finish_observed_request, + ) + app.teardown_request(_close_observed_request) + extensions[_EXTENSION_KEY] = runtime + except Exception as exc: + if snapshots is not None: + _restore_callback_registries(app, snapshots, runtime) + try: + if existing is _MISSING: + extensions.pop(_EXTENSION_KEY, None) + else: + extensions[_EXTENSION_KEY] = existing + except Exception as rollback_error: + runtime.diagnostics.report( + "flask.callback_install_rollback", rollback_error + ) + runtime.diagnostics.report("flask.callback_install", exc) + return runtime + + +def _move_callback_to_start( + registry: dict[Any, list[Any]], callback: Any +) -> None: + callbacks = registry.get(None) + if callbacks is None: + raise RuntimeError("Flask did not register the callback.") + for index, registered in enumerate(callbacks): + if registered is callback: + callbacks.insert(0, callbacks.pop(index)) + return + raise RuntimeError("Flask did not register the callback.") -def init_flask_observability( + +def _snapshot_callback_registries( app: Any, - *, - config: ObservabilityConfig | None = None, - runtime: ObservabilityRuntime | None = None, - service_name: str, - service_role: str = "api", - span_prefix: str | None = None, - segment_registry=None, -) -> ObservabilityRuntime: - if app.extensions.get("policyengine_observability"): - return app.extensions["policyengine_observability"] - runtime = runtime or ObservabilityRuntime( - config - or ObservabilityConfig.from_env( - service_name=service_name, - service_role=service_role, - span_prefix=span_prefix, - ), - segment_registry=segment_registry, - ) - runtime.configure() - app.extensions["policyengine_observability"] = runtime - set_observability_runtime(runtime) - FlaskObservabilityAdapter(runtime).instrument_app(app) - return runtime +) -> dict[str, dict[Any, list[Any]]]: + return { + name: { + key: list(callbacks) + for key, callbacks in getattr(app, name).items() + } + for name in _CALLBACK_REGISTRIES + } -def _split_forwarded_for(value: str | None) -> list[str]: - if not value: - return [] - return [part.strip() for part in value.split(",") if part.strip()] +def _restore_callback_registries( + app: Any, + snapshots: dict[str, dict[Any, list[Any]]], + runtime: ObservabilityRuntime, +) -> None: + for name, snapshot in snapshots.items(): + try: + registry = getattr(app, name) + registry.clear() + registry.update( + {key: list(callbacks) for key, callbacks in snapshot.items()} + ) + except Exception as exc: + runtime.diagnostics.report( + "flask.callback_install_rollback", + exc, + registry=name, + ) diff --git a/policyengine_observability/config.py b/policyengine_observability/config.py index 5d58448..6bada68 100644 --- a/policyengine_observability/config.py +++ b/policyengine_observability/config.py @@ -1,326 +1,653 @@ from __future__ import annotations -import logging +import math import os -from collections.abc import Mapping, Sequence -from dataclasses import dataclass -from typing import Any - -# Defaults for the generic queued-transport knobs; the queued destination -# imports these so a constructor call and an env-configured build can -# never disagree about what "default" means. -DEFAULT_LOG_QUEUE_MAXSIZE = 1000 -DEFAULT_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS = 2.0 - -DEFAULT_METRIC_ATTRIBUTE_KEYS = ( - "service.name", - "service.role", - "deployment.environment", - "operation", - "flavor", - "route", - "method", - "endpoint", - "status_code", - "country_id", - "backend", - "requested_version", - "resolved_channel", - "auth_result", - "segment", - "event", - "error_type", - "model", - "tool", - "stop_reason", - "iteration", - "provider", -) +from dataclasses import dataclass, field +from typing import Any, Literal, Protocol, cast +from .destinations import LogDestinationStrategy, StdoutLogDestination -def bool_from_env(name: str, default: bool) -> bool: - raw_value = os.getenv(name) - if raw_value is None: - return default - return raw_value.strip().lower() not in {"0", "false", "no", "off"} +Platform = Literal["google_cloud_run", "modal", "local", "other"] +OTLPProtocol = Literal["grpc", "http/protobuf"] +OTLPEndpointMode = Literal["base", "signal"] +ProviderMode = Literal["owned", "external"] -def csv_from_env(name: str) -> tuple[str, ...]: - raw_value = os.getenv(name) - if raw_value is None: - return () - return tuple(part.strip() for part in raw_value.split(",") if part.strip()) +class ConfigurationError(ValueError): + """Raised before startup when observability configuration is invalid.""" + def __init__(self, errors: tuple[str, ...]) -> None: + self.errors = errors + details = "\n".join(f"- {error}" for error in errors) + super().__init__(f"Invalid observability configuration:\n{details}") -def float_from_env(name: str, default: float) -> float: - raw_value = os.getenv(name) - if raw_value is None: - return default - try: - return float(raw_value) - except ValueError: - return default +DEFAULT_APPLICATION_ATTRIBUTE_KEYS = frozenset(set()) + +DEFAULT_DISPATCH_ATTRIBUTE_KEYS = frozenset(set()) + +DEFAULT_METRIC_ATTRIBUTE_KEYS = frozenset( + { + "service.name", + "service.role", + "deployment.environment.name", + "cloud.platform", + "http.route", + "http.request.method", + "http.response.status_code_class", + "operation.name", + "operation.kind", + "outcome", + } +) + + +@dataclass(frozen=True, slots=True) +class ServiceIdentity: + name: str + namespace: str + version: str + role: str -def int_from_env(name: str, default: int) -> int: - raw_value = os.getenv(name) - if raw_value is None: - return default - try: - return int(raw_value) - except ValueError: - return default +@dataclass(frozen=True, slots=True) +class DeploymentIdentity: + environment: str + platform: Platform + region: str | None = None + instance_id: str | None = None -def default_environment() -> str: - return ( - os.getenv("OBSERVABILITY_ENVIRONMENT") - or os.getenv("DEPLOYMENT_ENVIRONMENT") - or os.getenv("APP_ENV") - or os.getenv("ENVIRONMENT") - or "development" + +@dataclass(frozen=True, slots=True) +class LoggingConfig: + destinations: tuple[LogDestinationStrategy, ...] = field( + default_factory=lambda: (StdoutLogDestination(),) ) + capture_standard_library: bool = False + replace_existing_handlers: bool = False + minimum_severity: int = 20 + shutdown_timeout_seconds: float = 2.0 -# A log profile is a named preset expanding to generic routing primitives -# (a destination-name tuple and a stdout-formatter name). Presets may name -# strategies; the expansion mechanism knows nothing about any backend. -LOG_PROFILE_PRESETS: dict[str, tuple[tuple[str, ...], str]] = { - # Platforms whose logging agent ingests stdout (Cloud Run, GKE): - # agent-native stdout only, fully synchronous, zero threads. - "gcp-agent": (("stdout",), "google"), - # Platforms with no ingesting agent (Modal): plain stdout as the - # durable record plus queued direct Cloud Logging writes. - "gcp-direct": (("stdout", "google_cloud_logging"), "plain"), - # Local development and the kill switch: plain stdout, zero threads. - "plain-sync": (("stdout",), "plain"), -} - - -def _detect_log_profile() -> str | None: - platform = (os.getenv("OBSERVABILITY_PLATFORM") or "").strip().lower() - if platform == "google_cloud_run": - return "gcp-agent" - if platform == "modal": - return "gcp-direct" - if os.getenv("K_SERVICE"): - return "gcp-agent" - if os.getenv("MODAL_ENVIRONMENT") or os.getenv("MODAL_TASK_ID"): - return "gcp-direct" - return None - - -def _missing_strategy_requirements( - destination_names: Sequence[str], - resolved_config: Mapping[str, Any], -) -> list[tuple[str, str]]: - """(destination, config field) pairs a preset needs but lacks. - - Strategies declare their requirements at registration - (``register_destination(required_config=...)``); this check knows - nothing about any backend. - """ - # Imported lazily: the destinations package imports this module, so - # a module-level import here would be circular. By the time a config - # is resolved the package (and its strategy registrations) is loaded. - from .destinations.registry import destination_strategy - - missing: list[tuple[str, str]] = [] - for name in destination_names: - strategy = destination_strategy(name) - if strategy is None: - continue - for field in strategy.required_config: - if not resolved_config.get(field): - missing.append((name, field)) - return missing +class OTLPAuthentication(Protocol): + """Adds authentication-specific arguments to an OTLP exporter.""" + def exporter_kwargs( + self, + *, + protocol: OTLPProtocol, + headers: dict[str, str], + ) -> dict[str, Any]: ... -def _resolve_log_profile( - raw_profile: str, - *, - resolved_config: Mapping[str, Any], -) -> tuple[str, tuple[tuple[str, ...], str] | None, list[str]]: - """Resolve a profile name to (name, preset-or-None, warnings). - - ``auto`` without a recognized platform marker resolves to no preset, - so caller-supplied defaults keep applying. ``resolved_config`` - carries the already-resolved config values that registered - strategies may declare as requirements. - """ - warnings: list[str] = [] - # Canonical profile names are hyphenated; accept the same case, - # whitespace, and hyphen/underscore variance as destination and - # formatter names. - profile = raw_profile.strip().lower().replace("_", "-") - if profile == "auto": - detected = _detect_log_profile() - if detected is None: - return "auto", None, warnings - profile = detected - preset = LOG_PROFILE_PRESETS.get(profile) - if preset is None: - warnings.append( - f"Unknown OBSERVABILITY_LOG_PROFILE {raw_profile!r}; " - "using plain-sync." - ) - profile = "plain-sync" - preset = LOG_PROFILE_PRESETS[profile] - missing = _missing_strategy_requirements(preset[0], resolved_config) - if missing: - requirements = ", ".join( - f"{field} (destination {name})" for name, field in missing - ) - warnings.append( - f"Log profile {profile} requires {requirements}; using plain-sync." - ) - profile = "plain-sync" - preset = LOG_PROFILE_PRESETS[profile] - return profile, preset, warnings +@dataclass(frozen=True, slots=True) +class OTLPExporterConfig: + endpoint: str + protocol: OTLPProtocol = "grpc" + endpoint_mode: OTLPEndpointMode = "base" + headers: tuple[tuple[str, str], ...] = () + auth: OTLPAuthentication | None = None + timeout_seconds: float = 5.0 -@dataclass(frozen=True) -class ObservabilityConfig: - service_name: str = "policyengine-service" - service_role: str = "api" - environment: str = "development" + +@dataclass(frozen=True, slots=True) +class OTelConfig: enabled: bool = True - request_logs_enabled: bool = True - log_raw_ip: bool = True - log_level: int = logging.INFO - otel_enabled: bool = True - otlp_endpoint: str | None = None - otlp_protocol: str = "grpc" - span_prefix: str | None = None - tracer_name: str | None = None - meter_name: str | None = None + traces: OTLPExporterConfig | None = None + metrics: OTLPExporterConfig | None = None + provider_mode: ProviderMode = "owned" + sampling_ratio: float = 1.0 + span_queue_capacity: int = 2_048 + span_batch_size: int = 512 + span_schedule_delay_seconds: float = 5.0 + metric_export_interval_seconds: float = 60.0 shutdown_timeout_seconds: float = 3.0 - instrument_fastapi: bool = False - instrument_httpx: bool = False - metric_attribute_keys: tuple[str, ...] = DEFAULT_METRIC_ATTRIBUTE_KEYS - log_destinations: tuple[str, ...] = ("stdout",) - google_cloud_project: str | None = None - google_cloud_log_name: str = "policyengine-observability" - stdout_format: str = "plain" - log_queue_maxsize: int = DEFAULT_LOG_QUEUE_MAXSIZE - log_queue_close_timeout_seconds: float = ( - DEFAULT_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS + + +@dataclass(frozen=True, slots=True) +class TelemetryLimits: + max_attributes: int = 32 + max_string_length: int = 1_024 + max_error_message_length: int = 2_048 + max_stack_length: int = 16_384 + async_parent_max_age_seconds: float = 300.0 + + +@dataclass(frozen=True, slots=True) +class ObservabilityConfig: + service: ServiceIdentity + deployment: DeploymentIdentity + logging: LoggingConfig = field(default_factory=LoggingConfig) + otel: OTelConfig = field(default_factory=OTelConfig) + limits: TelemetryLimits = field(default_factory=TelemetryLimits) + application_attribute_keys: frozenset[str] = ( + DEFAULT_APPLICATION_ATTRIBUTE_KEYS ) - log_profile: str = "auto" - config_warnings: tuple[str, ...] = () + dispatch_attribute_keys: frozenset[str] = DEFAULT_DISPATCH_ATTRIBUTE_KEYS + metric_attribute_keys: frozenset[str] = DEFAULT_METRIC_ATTRIBUTE_KEYS + sensitive_values: tuple[str, ...] = () @classmethod def from_env( cls, *, - service_name: str, - service_role: str = "api", - enabled_default: bool = True, - span_prefix: str | None = None, - instrument_fastapi: bool = False, - instrument_httpx: bool = False, - metric_attribute_keys: Sequence[str] | None = None, - extra_metric_attribute_keys: Sequence[str] = (), - default_log_destinations: Sequence[str] = ("stdout",), + service: ServiceIdentity, + deployment: DeploymentIdentity, + logging: LoggingConfig | None = None, + limits: TelemetryLimits | None = None, + application_attribute_keys: frozenset[str] | None = None, + dispatch_attribute_keys: frozenset[str] | None = None, + metric_attribute_keys: frozenset[str] | None = None, + sensitive_values: tuple[str, ...] = (), ) -> ObservabilityConfig: - level_name = os.getenv("OBSERVABILITY_LOG_LEVEL", "INFO").upper() - log_level = getattr(logging, level_name, logging.INFO) - otlp_protocol = ( - os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL") - or os.getenv("OBSERVABILITY_OTLP_PROTOCOL") - or cls.otlp_protocol - ) - env_metric_keys = csv_from_env("OBSERVABILITY_METRIC_ATTRIBUTE_KEYS") - env_extra_metric_keys = csv_from_env( - "OBSERVABILITY_EXTRA_METRIC_ATTRIBUTE_KEYS" - ) - env_log_destinations = csv_from_env("OBSERVABILITY_LOG_DESTINATIONS") - resolved_metric_keys = _dedupe( - env_metric_keys - or metric_attribute_keys - or DEFAULT_METRIC_ATTRIBUTE_KEYS, - (*extra_metric_attribute_keys, *env_extra_metric_keys), - ) - google_cloud_project = ( - os.getenv("OBSERVABILITY_GOOGLE_CLOUD_PROJECT") - or os.getenv("GOOGLE_CLOUD_PROJECT") - or os.getenv("GCP_PROJECT") - or os.getenv("GCLOUD_PROJECT") - or None - ) - log_profile, preset, profile_warnings = _resolve_log_profile( - os.getenv("OBSERVABILITY_LOG_PROFILE") or cls.log_profile, - # The values strategies may declare via required_config; - # extend as future fields become requirement candidates. - resolved_config={"google_cloud_project": google_cloud_project}, + """Read standard OTel transport settings with explicit identity. + + Service identity, deployment identity, and log routing are never + inferred from ambient platform or Google Cloud variables. + """ + + common_endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") + common_protocol = _protocol( + "OTEL_EXPORTER_OTLP_PROTOCOL", + os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc"), ) - profile_destinations, profile_stdout_format = preset or (None, None) - # Explicit granular env vars override the profile's expansion; - # the profile overrides caller-supplied defaults. - resolved_log_destinations = _dedupe( - env_log_destinations - or profile_destinations - or default_log_destinations + common_headers = os.getenv("OTEL_EXPORTER_OTLP_HEADERS", "") + common_audience = os.getenv("POLICYENGINE_OTEL_GOOGLE_AUDIENCE") + traces = _exporter_from_env( + signal="traces", + enabled=_export_enabled("OTEL_TRACES_EXPORTER"), + common_endpoint=common_endpoint, + common_protocol=common_protocol, + common_headers=common_headers, + common_audience=common_audience, ) - resolved_stdout_format = ( - os.getenv("OBSERVABILITY_STDOUT_FORMAT") - or profile_stdout_format - or cls.stdout_format + metrics = _exporter_from_env( + signal="metrics", + enabled=_export_enabled("OTEL_METRICS_EXPORTER"), + common_endpoint=common_endpoint, + common_protocol=common_protocol, + common_headers=common_headers, + common_audience=common_audience, ) - return cls( - service_name=os.getenv("OBSERVABILITY_SERVICE_NAME") - or os.getenv("OTEL_SERVICE_NAME") - or service_name, - service_role=service_role, - environment=default_environment(), - enabled=bool_from_env("OBSERVABILITY_ENABLED", enabled_default), - request_logs_enabled=bool_from_env( - "OBSERVABILITY_REQUEST_LOGS_ENABLED", - True, - ), - log_raw_ip=bool_from_env("OBSERVABILITY_LOG_RAW_IP", True), - log_level=log_level, - otel_enabled=bool_from_env("OTEL_ENABLED", cls.otel_enabled), - otlp_endpoint=os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT") or None, - otlp_protocol=otlp_protocol, - span_prefix=span_prefix, - tracer_name=os.getenv("OBSERVABILITY_TRACER_NAME"), - meter_name=os.getenv("OBSERVABILITY_METER_NAME"), - shutdown_timeout_seconds=float_from_env( - "OBSERVABILITY_SHUTDOWN_TIMEOUT_SECONDS", - cls.shutdown_timeout_seconds, - ), - instrument_fastapi=bool_from_env( - "OBSERVABILITY_INSTRUMENT_FASTAPI", - instrument_fastapi, - ), - instrument_httpx=bool_from_env( - "OBSERVABILITY_INSTRUMENT_HTTPX", - instrument_httpx, + + config = cls( + service=service, + deployment=deployment, + logging=logging or LoggingConfig(), + otel=OTelConfig( + enabled=not _env_bool("OTEL_SDK_DISABLED", False), + traces=traces, + metrics=metrics, + provider_mode=_provider_mode( + os.getenv("POLICYENGINE_OTEL_PROVIDER_MODE", "owned") + ), + sampling_ratio=_env_float( + "OTEL_TRACES_SAMPLER_ARG", + os.getenv("OTEL_TRACES_SAMPLER_ARG"), + default=1.0, + minimum=0.0, + maximum=1.0, + ), + span_queue_capacity=_env_int( + "OTEL_BSP_MAX_QUEUE_SIZE", + os.getenv("OTEL_BSP_MAX_QUEUE_SIZE"), + default=2_048, + minimum=1, + maximum=100_000, + ), + span_batch_size=_env_int( + "OTEL_BSP_MAX_EXPORT_BATCH_SIZE", + os.getenv("OTEL_BSP_MAX_EXPORT_BATCH_SIZE"), + default=512, + minimum=1, + maximum=10_000, + ), + span_schedule_delay_seconds=( + _env_float( + "OTEL_BSP_SCHEDULE_DELAY", + os.getenv("OTEL_BSP_SCHEDULE_DELAY"), + default=5_000.0, + minimum=1.0, + maximum=60_000.0, + ) + / 1_000 + ), + metric_export_interval_seconds=( + _env_float( + "OTEL_METRIC_EXPORT_INTERVAL", + os.getenv("OTEL_METRIC_EXPORT_INTERVAL"), + default=60_000.0, + minimum=1_000.0, + maximum=3_600_000.0, + ) + / 1_000 + ), ), - metric_attribute_keys=resolved_metric_keys, - log_destinations=resolved_log_destinations, - google_cloud_project=google_cloud_project, - google_cloud_log_name=( - os.getenv("OBSERVABILITY_GOOGLE_CLOUD_LOG_NAME") - or cls.google_cloud_log_name + limits=limits or TelemetryLimits(), + application_attribute_keys=( + application_attribute_keys + if application_attribute_keys is not None + else DEFAULT_APPLICATION_ATTRIBUTE_KEYS ), - stdout_format=resolved_stdout_format, - log_queue_maxsize=int_from_env( - "OBSERVABILITY_LOG_QUEUE_MAXSIZE", - cls.log_queue_maxsize, + dispatch_attribute_keys=( + dispatch_attribute_keys + if dispatch_attribute_keys is not None + else DEFAULT_DISPATCH_ATTRIBUTE_KEYS ), - log_queue_close_timeout_seconds=float_from_env( - "OBSERVABILITY_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS", - cls.log_queue_close_timeout_seconds, + metric_attribute_keys=( + metric_attribute_keys + if metric_attribute_keys is not None + else DEFAULT_METRIC_ATTRIBUTE_KEYS ), - log_profile=log_profile, - config_warnings=tuple(profile_warnings), + sensitive_values=sensitive_values, + ) + config.validate() + return config + + def validate(self) -> None: + """Reject invalid configuration before runtime setup has side effects.""" + + errors = self.validation_errors() + if errors: + raise ConfigurationError(errors) + + def validation_errors(self) -> tuple[str, ...]: + """Return invalid fields without creating runtime components.""" + + errors: list[str] = [] + identity_values = { + "service.name": self.service.name, + "service.namespace": self.service.namespace, + "service.version": self.service.version, + "service.role": self.service.role, + "deployment.environment": self.deployment.environment, + } + for key, value in identity_values.items(): + if not isinstance(value, str) or not value.strip(): + errors.append(f"{key} must be a non-empty string.") + + if not isinstance(self.sensitive_values, tuple): + errors.append( + "sensitive_values must be a tuple of non-empty strings." + ) + else: + for index, value in enumerate(self.sensitive_values): + if not isinstance(value, str) or not value.strip(): + errors.append( + f"sensitive_values[{index}] must be a non-empty " + "string." + ) + + for name, values in ( + ("application_attribute_keys", self.application_attribute_keys), + ("dispatch_attribute_keys", self.dispatch_attribute_keys), + ("metric_attribute_keys", self.metric_attribute_keys), + ): + if not isinstance(values, frozenset): + errors.append( + f"{name} must be a frozenset of non-empty strings." + ) + continue + for value in values: + if not isinstance(value, str) or not value.strip(): + errors.append(f"{name} entries must be non-empty strings.") + + _choice_error( + errors, + "deployment.platform", + self.deployment.platform, + {"google_cloud_run", "modal", "local", "other"}, + ) + if not isinstance(self.logging.minimum_severity, int) or isinstance( + self.logging.minimum_severity, bool + ): + errors.append("logging.minimum_severity must be an integer.") + _number_error( + errors, + "logging.shutdown_timeout_seconds", + self.logging.shutdown_timeout_seconds, + 0, + 60, + ) + + for destination in self.logging.destinations: + try: + _choice_error( + errors, + f"logging destination {destination.name!r} delivery", + destination.delivery, + {"inline", "queued"}, + ) + _integer_error( + errors, + f"logging destination {destination.name!r} queue_capacity", + destination.queue_capacity, + 1, + 100_000, + ) + _integer_error( + errors, + f"logging destination {destination.name!r} batch_size", + destination.batch_size, + 1, + 10_000, + ) + validate = getattr(destination, "diagnostics", None) + if callable(validate): + errors.extend( + str(item) for item in cast(tuple[str, ...], validate()) + ) + except Exception as exc: + errors.append( + "Invalid log destination strategy " + f"{getattr(destination, 'name', '')}: {exc}" + ) + + _choice_error( + errors, + "otel.provider_mode", + self.otel.provider_mode, + {"owned", "external"}, ) + _number_error( + errors, "otel.sampling_ratio", self.otel.sampling_ratio, 0, 1 + ) + _integer_error( + errors, + "otel.span_queue_capacity", + self.otel.span_queue_capacity, + 1, + 100_000, + ) + _integer_error( + errors, + "otel.span_batch_size", + self.otel.span_batch_size, + 1, + 10_000, + ) + _number_error( + errors, + "otel.span_schedule_delay_seconds", + self.otel.span_schedule_delay_seconds, + 0.001, + 60, + ) + _number_error( + errors, + "otel.metric_export_interval_seconds", + self.otel.metric_export_interval_seconds, + 1, + 3_600, + ) + _number_error( + errors, + "otel.shutdown_timeout_seconds", + self.otel.shutdown_timeout_seconds, + 0, + 60, + ) + for signal, exporter in ( + ("traces", self.otel.traces), + ("metrics", self.otel.metrics), + ): + if exporter is None: + continue + if ( + not isinstance(exporter.endpoint, str) + or not exporter.endpoint.strip() + ): + errors.append( + f"otel.{signal}.endpoint must be a non-empty string." + ) + _choice_error( + errors, + f"otel.{signal}.protocol", + exporter.protocol, + {"grpc", "http/protobuf"}, + ) + _choice_error( + errors, + f"otel.{signal}.endpoint_mode", + exporter.endpoint_mode, + {"base", "signal"}, + ) + _number_error( + errors, + f"otel.{signal}.timeout_seconds", + exporter.timeout_seconds, + 0.1, + 60, + ) + + for name, value in { + "limits.max_attributes": self.limits.max_attributes, + "limits.max_string_length": self.limits.max_string_length, + "limits.max_error_message_length": self.limits.max_error_message_length, + "limits.max_stack_length": self.limits.max_stack_length, + }.items(): + _integer_error(errors, name, value, 1, 1_000_000) + _number_error( + errors, + "limits.async_parent_max_age_seconds", + self.limits.async_parent_max_age_seconds, + 0, + 86_400, + ) + return tuple(errors) + + def diagnostics(self) -> tuple[str, ...]: + messages: list[str] = [] + if ( + self.otel.enabled + and self.otel.provider_mode == "owned" + and self.otel.traces is None + and self.otel.metrics is None + ): + messages.append( + "Remote OTel export is disabled because no trace or metric " + "OTLP endpoint is configured." + ) + return tuple(messages) + + @property + def identity_complete(self) -> bool: + return not any( + not str(value).strip() + for value in ( + self.service.name, + self.service.namespace, + self.service.version, + self.service.role, + self.deployment.environment, + self.deployment.platform, + ) + ) + + +def _env_bool(name: str, default: bool) -> bool: + value = os.getenv(name) + if value is None: + return default + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + raise ConfigurationError( + (f"{name} must be a boolean value; received {value!r}.",) + ) + + +def _provider_mode(value: str) -> ProviderMode: + normalized = value.strip().lower() + if normalized not in {"owned", "external"}: + raise ConfigurationError( + (f"POLICYENGINE_OTEL_PROVIDER_MODE is invalid: {value!r}.",) + ) + return cast(ProviderMode, normalized) + + +def _protocol(name: str, value: str) -> OTLPProtocol: + normalized = value.strip() + if normalized not in {"grpc", "http/protobuf"}: + raise ConfigurationError((f"{name} is invalid: {value!r}.",)) + return cast(OTLPProtocol, normalized) + + +def _export_enabled(name: str) -> bool: + value = os.getenv(name, "otlp").strip().lower() + if value not in {"otlp", "none"}: + raise ConfigurationError((f"{name} is invalid: {value!r}.",)) + return value == "otlp" + + +def _exporter_from_env( + *, + signal: Literal["traces", "metrics"], + enabled: bool, + common_endpoint: str | None, + common_protocol: OTLPProtocol, + common_headers: str, + common_audience: str | None, +) -> OTLPExporterConfig | None: + if not enabled: + return None + prefix = f"OTEL_EXPORTER_OTLP_{signal.upper()}" + signal_endpoint = os.getenv(f"{prefix}_ENDPOINT") + endpoint = signal_endpoint or common_endpoint + if not endpoint: + return None + protocol_value = os.getenv(f"{prefix}_PROTOCOL") + protocol = ( + _protocol(f"{prefix}_PROTOCOL", protocol_value) + if protocol_value is not None + else common_protocol + ) + headers_value = os.getenv(f"{prefix}_HEADERS") + headers = _parse_headers( + f"{prefix}_HEADERS" + if headers_value is not None + else "OTEL_EXPORTER_OTLP_HEADERS", + headers_value if headers_value is not None else common_headers, + ) + audience = ( + os.getenv(f"POLICYENGINE_OTEL_{signal.upper()}_GOOGLE_AUDIENCE") + or common_audience + ) + auth: OTLPAuthentication | None = None + if audience: + from .google_auth import GoogleIdTokenAuth + + auth = GoogleIdTokenAuth(audience) + signal_timeout = os.getenv(f"{prefix}_TIMEOUT") + timeout_name = ( + f"{prefix}_TIMEOUT" + if signal_timeout is not None + else "OTEL_EXPORTER_OTLP_TIMEOUT" + ) + timeout = _env_float( + timeout_name, + signal_timeout or os.getenv("OTEL_EXPORTER_OTLP_TIMEOUT"), + default=5_000.0, + minimum=100.0, + maximum=60_000.0, + ) + return OTLPExporterConfig( + endpoint=endpoint, + protocol=protocol, + endpoint_mode="signal" if signal_endpoint else "base", + headers=headers, + auth=auth, + timeout_seconds=timeout / 1_000, + ) + + +def _parse_headers(name: str, value: str) -> tuple[tuple[str, str], ...]: + headers: list[tuple[str, str]] = [] + for item in value.split(","): + if not item.strip(): + continue + if "=" not in item: + raise ConfigurationError( + (f"{name} contains a header without '=': {item!r}.",) + ) + key, header_value = item.split("=", 1) + if not key.strip(): + raise ConfigurationError( + (f"{name} contains an empty header name.",) + ) + headers.append((key.strip(), header_value.strip())) + return tuple(headers) + + +def _env_float( + name: str, + value: str | None, + *, + default: float, + minimum: float, + maximum: float, +) -> float: + try: + parsed = float(value) if value is not None else default + except (TypeError, ValueError): + raise ConfigurationError( + (f"{name} must be a number; received {value!r}.",) + ) from None + if not math.isfinite(parsed) or not minimum <= parsed <= maximum: + raise ConfigurationError( + (f"{name} must be between {minimum} and {maximum}.",) + ) + return parsed + + +def _env_int( + name: str, + value: str | None, + *, + default: int, + minimum: int, + maximum: int, +) -> int: + try: + parsed = int(value) if value is not None else default + except (TypeError, ValueError): + raise ConfigurationError( + (f"{name} must be an integer; received {value!r}.",) + ) from None + if not minimum <= parsed <= maximum: + raise ConfigurationError( + (f"{name} must be between {minimum} and {maximum}.",) + ) + return parsed + + +def _choice_error( + errors: list[str], name: str, value: Any, choices: set[str] +) -> None: + if not isinstance(value, str) or value not in choices: + errors.append(f"{name} must be one of: {', '.join(sorted(choices))}.") + + +def _number_error( + errors: list[str], name: str, value: Any, minimum: float, maximum: float +) -> None: + if ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not math.isfinite(value) + or not minimum <= value <= maximum + ): + errors.append(f"{name} must be between {minimum} and {maximum}.") -def _dedupe( - base: Sequence[str], - extra: Sequence[str] = (), -) -> tuple[str, ...]: - return tuple(dict.fromkeys((*base, *extra))) +def _integer_error( + errors: list[str], name: str, value: Any, minimum: int, maximum: int +) -> None: + if ( + not isinstance(value, int) + or isinstance(value, bool) + or not minimum <= value <= maximum + ): + errors.append(f"{name} must be between {minimum} and {maximum}.") diff --git a/policyengine_observability/context.py b/policyengine_observability/context.py deleted file mode 100644 index bb2d464..0000000 --- a/policyengine_observability/context.py +++ /dev/null @@ -1,270 +0,0 @@ -from __future__ import annotations - -import time -from dataclasses import dataclass, field -from datetime import UTC, datetime -from typing import Any - -from .config import ObservabilityConfig - - -@dataclass -class ErrorRecord: - type: str - message: str - handled: bool - stack: str | None = None - - def as_dict(self) -> dict[str, Any]: - return { - "type": self.type, - "message": self.message, - "handled": self.handled, - "stack": self.stack, - } - - -@dataclass -class SegmentTimingNode: - sequence: int - name: str - attrs: dict[str, Any] = field(default_factory=dict) - duration_ms: float | None = None - children: list[SegmentTimingNode] = field(default_factory=list) - - def as_dict(self) -> dict[str, Any]: - record: dict[str, Any] = { - "sequence": self.sequence, - "name": self.name, - } - if self.attrs: - record["attrs"] = dict(self.attrs) - if self.duration_ms is not None: - record["duration_ms"] = round(self.duration_ms, 3) - if self.children: - record["children"] = [child.as_dict() for child in self.children] - return record - - -@dataclass -class OperationObservabilityContext: - config: ObservabilityConfig - name: str - flavor: str | None = None - attributes: dict[str, Any] = field(default_factory=dict) - timings_ms: dict[str, float] = field(default_factory=dict) - timing_counts: dict[str, int] = field(default_factory=dict) - segment_tree: list[SegmentTimingNode] = field(default_factory=list) - segment_sequence: list[int] = field(default_factory=lambda: [0]) - emit_log: bool = True - record_metric: bool = True - started_at: float = field(default_factory=time.perf_counter) - created_at: datetime = field(default_factory=lambda: datetime.now(UTC)) - error: ErrorRecord | None = None - emitted: bool = False - metric_recorded: bool = False - span_handle: Any = None - context_token: Any = None - - def set_attribute(self, key: str, value: Any) -> None: - if value is None: - return - if hasattr(value, "value"): - value = value.value - self.attributes[key] = value - - def duration_seconds(self) -> float: - return time.perf_counter() - self.started_at - - def metric_attributes(self, **extra: Any) -> dict[str, str]: - attrs: dict[str, Any] = { - "service.name": self.config.service_name, - "service.role": self.config.service_role, - "deployment.environment": self.config.environment, - "operation": self.name, - "flavor": self.flavor, - } - for key in self.config.metric_attribute_keys: - if key in self.attributes: - attrs[key] = self.attributes[key] - attrs.update(extra) - return _metric_attrs(attrs, self.config.metric_attribute_keys) - - def span_attributes(self, **extra: Any) -> dict[str, Any]: - attrs: dict[str, Any] = { - "service.name": self.config.service_name, - "service.role": self.config.service_role, - "deployment.environment": self.config.environment, - "policyengine.operation": self.name, - "policyengine.flavor": self.flavor, - } - attrs.update( - { - f"policyengine.{key}": value - for key, value in self.attributes.items() - if value is not None - } - ) - attrs.update(extra) - return { - key: value for key, value in attrs.items() if value is not None - } - - def as_log_record( - self, - *, - trace_id: str | None, - span_id: str | None, - ) -> dict[str, Any]: - event = "operation_failed" if self.error else "operation_completed" - return { - **self.attributes, - "schema_version": "policyengine.observability.operation.v1", - "event": event, - "service_name": self.config.service_name, - "service_role": self.config.service_role, - "environment": self.config.environment, - "created_at": self.created_at.isoformat(), - "operation": self.name, - "flavor": self.flavor, - "trace_id": trace_id, - "span_id": span_id, - "duration_ms": round(self.duration_seconds() * 1000, 3), - "timings_ms": dict(self.timings_ms), - "timing_counts": dict(self.timing_counts), - "segment_tree": [node.as_dict() for node in self.segment_tree], - "error": self.error.as_dict() if self.error else None, - } - - -@dataclass -class RequestObservabilityContext: - config: ObservabilityConfig - request_id: str - method: str - route: str - path: str - endpoint: str | None - query_keys: list[str] - content_length_bytes: int | None - inbound: dict[str, Any] - internal_dispatch: bool = False - started_at: float = field(default_factory=time.perf_counter) - created_at: datetime = field(default_factory=lambda: datetime.now(UTC)) - attributes: dict[str, Any] = field(default_factory=dict) - timings_ms: dict[str, float] = field(default_factory=dict) - timing_counts: dict[str, int] = field(default_factory=dict) - segment_tree: list[SegmentTimingNode] = field(default_factory=list) - segment_sequence: list[int] = field(default_factory=lambda: [0]) - status_code: int | None = None - error: ErrorRecord | None = None - emitted: bool = False - request_metric_recorded: bool = False - active_closed: bool = False - span_closed: bool = False - server_span_cm: Any = None - server_span: Any = None - context_token: Any = None - operation_context: OperationObservabilityContext | None = None - operation_token: Any = None - - def set_attribute(self, key: str, value: Any) -> None: - if value is None: - return - if hasattr(value, "value"): - value = value.value - self.attributes[key] = value - - def duration_seconds(self) -> float: - return time.perf_counter() - self.started_at - - def metric_attributes(self, **extra: Any) -> dict[str, str]: - attrs: dict[str, Any] = { - "service.name": self.config.service_name, - "service.role": self.config.service_role, - "deployment.environment": self.config.environment, - "route": self.route, - "method": self.method, - "endpoint": self.endpoint, - } - if self.status_code is not None: - attrs["status_code"] = str(self.status_code) - for key in self.config.metric_attribute_keys: - if key in self.attributes: - attrs[key] = self.attributes[key] - attrs.update(extra) - return _metric_attrs(attrs, self.config.metric_attribute_keys) - - def span_attributes(self, **extra: Any) -> dict[str, Any]: - attrs: dict[str, Any] = { - "service.name": self.config.service_name, - "service.role": self.config.service_role, - "deployment.environment": self.config.environment, - "http.request.method": self.method, - "http.route": self.route, - "url.path": self.path, - "policyengine.endpoint": self.endpoint, - "policyengine.request_id": self.request_id, - } - if self.status_code is not None: - attrs["http.response.status_code"] = self.status_code - for key in ( - "country_id", - "backend", - "requested_version", - "resolved_channel", - "auth_result", - ): - if key in self.attributes: - attrs[f"policyengine.{key}"] = self.attributes[key] - attrs.update(extra) - return { - key: value for key, value in attrs.items() if value is not None - } - - def as_log_record( - self, - *, - trace_id: str | None, - span_id: str | None, - ) -> dict[str, Any]: - event = ( - "http_request_failed" if self.error else "http_request_completed" - ) - status_code = self.status_code or (500 if self.error else None) - return { - **self.inbound, - **self.attributes, - "schema_version": "policyengine.observability.request.v1", - "event": event, - "service_name": self.config.service_name, - "service_role": self.config.service_role, - "environment": self.config.environment, - "created_at": self.created_at.isoformat(), - "request_id": self.request_id, - "trace_id": trace_id, - "span_id": span_id, - "method": self.method, - "route": self.route, - "path": self.path, - "query_keys": self.query_keys, - "endpoint": self.endpoint, - "status_code": status_code, - "duration_ms": round(self.duration_seconds() * 1000, 3), - "timings_ms": dict(self.timings_ms), - "timing_counts": dict(self.timing_counts), - "segment_tree": [node.as_dict() for node in self.segment_tree], - "error": self.error.as_dict() if self.error else None, - } - - -def _metric_attrs( - attrs: dict[str, Any], - metric_attribute_keys: tuple[str, ...], -) -> dict[str, str]: - result: dict[str, str] = {} - for key in metric_attribute_keys: - value = attrs.get(key) - if value is not None: - result[key] = str(value) - return result diff --git a/policyengine_observability/delivery.py b/policyengine_observability/delivery.py new file mode 100644 index 0000000..4facaab --- /dev/null +++ b/policyengine_observability/delivery.py @@ -0,0 +1,354 @@ +from __future__ import annotations + +import queue +import sys +import threading +import time +from copy import deepcopy +from dataclasses import dataclass +from typing import Any, TextIO + +from .config import ObservabilityConfig +from .destinations import ( + DestinationBuildContext, + LogDestinationStrategy, + RecordWriter, + StdoutLogDestination, +) +from .diagnostics import Diagnostics + +DESTINATION_FAILURE_LIMIT = 3 + + +@dataclass(slots=True) +class _InlineDestination: + strategy: LogDestinationStrategy + writer: RecordWriter + failures: int = 0 + + +@dataclass(slots=True) +class _WriterCleanup: + name: str + thread: threading.Thread + timeout_reported: bool = False + + +class DeliveryManager: + """Fan out provider-neutral records to independently isolated writers.""" + + def __init__( + self, + config: ObservabilityConfig, + diagnostics: Diagnostics, + *, + stdout: TextIO | None = None, + ) -> None: + self.config = config + self.diagnostics = diagnostics + self._stdout = stdout or sys.stdout + self._inline: list[_InlineDestination] = [] + self._queued: list[_QueuedWriter] = [] + self._cleanup_lock = threading.Lock() + self._cleanup_tasks: list[_WriterCleanup] = [] + self._configure() + + def _configure(self) -> None: + strategies = self.config.logging.destinations + if not strategies: + strategies = (StdoutLogDestination(),) + self.diagnostics.report( + "logging.destination_config", + "No log destination was configured; using standard output.", + ) + context = DestinationBuildContext(stdout=lambda: self._stdout) + for strategy in strategies: + try: + self._add_strategy(strategy, context) + except Exception as exc: + self.diagnostics.report( + "logging.destination_config", + exc, + destination=getattr(strategy, "name", None), + ) + if not self._inline and not self._queued: + self._add_strategy(StdoutLogDestination(), context) + + def _add_strategy( + self, + strategy: LogDestinationStrategy, + context: DestinationBuildContext, + ) -> None: + if strategy.delivery == "inline": + self._inline.append( + _InlineDestination(strategy, strategy.build_writer(context)) + ) + return + if strategy.delivery != "queued": + raise ValueError( + f"Unsupported delivery mode for {strategy.name!r}: " + f"{strategy.delivery!r}" + ) + self._queued.append( + _QueuedWriter( + strategy, + context, + self.diagnostics, + ) + ) + + def emit(self, record: dict[str, Any]) -> None: + disabled: list[_InlineDestination] = [] + for destination in tuple(self._inline): + try: + destination.writer.write(deepcopy(record)) + destination.failures = 0 + except Exception as exc: + destination.failures += 1 + self.diagnostics.increment("logs.export_failure") + self.diagnostics.report( + "logging.export", + exc, + destination=destination.strategy.name, + consecutive_failures=destination.failures, + ) + if destination.failures >= DESTINATION_FAILURE_LIMIT: + disabled.append(destination) + for destination in disabled: + self._disable_inline(destination) + for destination in tuple(self._queued): + destination.enqueue(deepcopy(record)) + + def _disable_inline(self, destination: _InlineDestination) -> None: + try: + self._inline.remove(destination) + except ValueError: + return + self._start_writer_cleanup( + destination.writer, + destination.strategy.name, + ) + self.diagnostics.report( + "logging.destination_disabled", + RuntimeError( + "Log destination disabled after repeated write failures." + ), + destination=destination.strategy.name, + ) + if self._inline or self._queued: + return + try: + context = DestinationBuildContext(stdout=lambda: self._stdout) + fallback = StdoutLogDestination() + self._inline.append( + _InlineDestination(fallback, fallback.build_writer(context)) + ) + except Exception as exc: + self.diagnostics.report("logging.stdout_fallback", exc) + + def close(self, timeout_seconds: float | None = None) -> None: + timeout = ( + self.config.logging.shutdown_timeout_seconds + if timeout_seconds is None + else timeout_seconds + ) + try: + timeout = min(max(float(timeout), 0.0), 30.0) + except (TypeError, ValueError): + timeout = 2.0 + deadline = time.monotonic() + timeout + inline = tuple(self._inline) + self._inline.clear() + for destination in inline: + self._start_writer_cleanup( + destination.writer, + destination.strategy.name, + ) + for destination in tuple(self._queued): + destination.close(max(0.0, deadline - time.monotonic())) + with self._cleanup_lock: + cleanup_tasks = tuple(self._cleanup_tasks) + for task in cleanup_tasks: + task.thread.join(max(0.0, deadline - time.monotonic())) + should_report_timeout = False + with self._cleanup_lock: + if task.thread.is_alive() and not task.timeout_reported: + task.timeout_reported = True + should_report_timeout = True + if should_report_timeout: + self.diagnostics.increment("logs.shutdown_timeout") + self.diagnostics.report( + "logging.shutdown_timeout", + "Log writer cleanup did not stop before its deadline.", + destination=task.name, + ) + + def _start_writer_cleanup(self, writer: RecordWriter, name: str) -> None: + close = getattr(writer, "close", None) + if not callable(close): + return + + def run() -> None: + try: + close() + except Exception as exc: + self.diagnostics.report( + "logging.writer_close", exc, destination=name + ) + + task = _WriterCleanup( + name=name, + thread=threading.Thread( + target=run, + name=f"policyengine-observability-close-{name}", + daemon=True, + ), + ) + try: + with self._cleanup_lock: + task.thread.start() + self._cleanup_tasks.append(task) + except Exception as exc: + self.diagnostics.report( + "logging.writer_close", exc, destination=name + ) + + @property + def remote_enabled(self) -> bool: + return bool(self._queued) + + @property + def queue_depth(self) -> int: + return sum(destination.queue_depth for destination in self._queued) + + +class _QueuedWriter: + _STOP = object() + + def __init__( + self, + strategy: LogDestinationStrategy, + context: DestinationBuildContext, + diagnostics: Diagnostics, + ) -> None: + self.strategy = strategy + self.context = context + self.diagnostics = diagnostics + capacity = max( + 1, + min(int(getattr(strategy, "queue_capacity", 1_000)), 100_000), + ) + self._queue: queue.Queue[dict[str, Any] | object] = queue.Queue( + maxsize=capacity + ) + self._closed = threading.Event() + self._writer: RecordWriter | None = None + self._thread = threading.Thread( + target=self._run, + name=f"policyengine-observability-{strategy.name}", + daemon=True, + ) + self._thread.start() + + def enqueue(self, record: dict[str, Any]) -> None: + if self._closed.is_set(): + self.diagnostics.increment("logs.dropped.closed") + return + try: + self._queue.put_nowait(record) + except queue.Full: + self.diagnostics.increment("logs.dropped.queue_full") + self.diagnostics.report( + "logging.queue_full", + "Remote log queue is full; newest record dropped.", + destination=self.strategy.name, + capacity=self._queue.maxsize, + ) + + @property + def queue_depth(self) -> int: + return self._queue.qsize() + + def close(self, timeout_seconds: float) -> None: + if self._closed.is_set(): + return + self._closed.set() + try: + self._queue.put_nowait(self._STOP) + except queue.Full: + pass + self._thread.join(max(0.0, timeout_seconds)) + if self._thread.is_alive(): + self.diagnostics.increment("logs.shutdown_timeout") + self.diagnostics.report( + "logging.shutdown_timeout", + "Remote log worker did not stop before its deadline.", + destination=self.strategy.name, + ) + + def _run(self) -> None: + try: + while True: + try: + item = self._queue.get(timeout=0.2) + except queue.Empty: + if self._closed.is_set(): + return + continue + consumed = [item] + stop_after_batch = item is self._STOP + batch: list[dict[str, Any]] = [] + if isinstance(item, dict): + batch.append(item) + batch_size = max( + 1, + min( + int(getattr(self.strategy, "batch_size", 1)), + 10_000, + ), + ) + while not stop_after_batch and len(batch) < batch_size: + try: + next_item = self._queue.get_nowait() + except queue.Empty: + break + consumed.append(next_item) + if next_item is self._STOP: + stop_after_batch = True + elif isinstance(next_item, dict): + batch.append(next_item) + try: + if not batch: + return + if self._writer is None: + self._writer = self.strategy.build_writer(self.context) + write_many = getattr(self._writer, "write_many", None) + if callable(write_many): + write_many(batch) + else: + for record in batch: + self._writer.write(record) + except Exception as exc: + self.diagnostics.increment("logs.export_failure") + self.diagnostics.report( + "logging.export", + exc, + destination=self.strategy.name, + ) + finally: + for _item in consumed: + self._queue.task_done() + if stop_after_batch: + return + finally: + if self._writer is not None: + close = getattr(self._writer, "close", None) + if callable(close): + try: + close() + except Exception as exc: + self.diagnostics.report( + "logging.writer_close", + exc, + destination=self.strategy.name, + ) diff --git a/policyengine_observability/destinations/__init__.py b/policyengine_observability/destinations/__init__.py index e37847b..f827215 100644 --- a/policyengine_observability/destinations/__init__.py +++ b/policyengine_observability/destinations/__init__.py @@ -1,19 +1,25 @@ -from __future__ import annotations +"""Logging destination strategies supplied to :class:`LoggingConfig`.""" -from .base import LogDestination, normalize_payload -from .google_cloud_logging import GoogleCloudLoggingDestination -from .manager import LogDestinationManager -from .queued import QueuedLogDestination -from .registry import register_destination -from .stdout import StdoutJsonDestination, register_stdout_formatter +from .base import ( + CustomLogDestination, + DestinationBuildContext, + LogDestinationStrategy, + RecordFormatter, + RecordWriter, +) +from .google_cloud import ( + GoogleCloudLogDestination, + GoogleCloudLogFormatter, +) +from .stdout import StdoutLogDestination __all__ = [ - "GoogleCloudLoggingDestination", - "LogDestination", - "LogDestinationManager", - "QueuedLogDestination", - "StdoutJsonDestination", - "normalize_payload", - "register_destination", - "register_stdout_formatter", + "CustomLogDestination", + "DestinationBuildContext", + "GoogleCloudLogDestination", + "GoogleCloudLogFormatter", + "LogDestinationStrategy", + "RecordFormatter", + "RecordWriter", + "StdoutLogDestination", ] diff --git a/policyengine_observability/destinations/base.py b/policyengine_observability/destinations/base.py index 8c93e09..b8a8323 100644 --- a/policyengine_observability/destinations/base.py +++ b/policyengine_observability/destinations/base.py @@ -1,121 +1,105 @@ from __future__ import annotations -import inspect -import math -from collections.abc import Callable, Mapping, Sequence -from typing import Any, Protocol +from collections.abc import Callable +from copy import deepcopy +from dataclasses import dataclass +from typing import Any, Literal, Protocol, TextIO, runtime_checkable +DeliveryMode = Literal["inline", "queued"] +RecordFormatter = Callable[[dict[str, Any]], dict[str, Any]] -class LogDestination(Protocol): - name: str - def emit( - self, - payload: dict[str, Any], - *, - log_type: str, - severity: str, - ) -> None: - """Write one structured observability payload.""" +class RecordWriter(Protocol): + """Writes provider-neutral structured records to one destination.""" + def write(self, record: dict[str, Any]) -> None: ... -def normalize_name(name: str) -> str: - """Canonical lookup key for registered names. - Destination, formatter, and profile lookups all forgive case, - surrounding whitespace, and hyphen/underscore variance the same way, - so a spelling that works for one registry works for every registry. - """ - return name.strip().lower().replace("-", "_") - - -def accepts_keyword(func: Callable[..., Any], name: str) -> bool: - """Whether ``func`` can safely be called with keyword ``name``.""" - try: - parameters = inspect.signature(func).parameters - except (TypeError, ValueError): - return False - if name in parameters: - return True - return any( - parameter.kind is inspect.Parameter.VAR_KEYWORD - for parameter in parameters.values() - ) - - -def safe_report( - on_failure: Callable[..., None], - operation: str, - exc: BaseException, - **fields: Any, -) -> None: - """Report through the internal-error channel; never raises.""" - try: - on_failure(operation, exc, **fields) - except Exception: - pass - - -def close_destination( - destination: LogDestination, - *, - on_failure: Callable[..., None], - deadline_seconds: float | None = None, -) -> None: - """Close a destination if it supports closing; never raises. - - ``close`` is duck-typed with one calling convention everywhere: the - deadline is passed only when the signature accepts it, so both - ``close(self)`` and ``close(self, deadline_seconds=None)`` work under - every close path (manager shutdown, reconfigure, queued drain). - """ - close = getattr(destination, "close", None) - if not callable(close): - return - try: - if accepts_keyword(close, "deadline_seconds"): - close(deadline_seconds=deadline_seconds) - else: +@dataclass(frozen=True, slots=True) +class DestinationBuildContext: + """Process-local resources available while constructing a writer.""" + + stdout: Callable[[], TextIO] + + +@runtime_checkable +class LogDestinationStrategy(Protocol): + """Configuration and factory contract for a logging destination.""" + + @property + def name(self) -> str: ... + + @property + def delivery(self) -> DeliveryMode: ... + + @property + def queue_capacity(self) -> int: ... + + @property + def batch_size(self) -> int: ... + + def build_writer( + self, context: DestinationBuildContext + ) -> RecordWriter: ... + + +class _FormattingWriter: + def __init__( + self, + writer: RecordWriter, + formatter: RecordFormatter | None, + ) -> None: + self._writer = writer + self._formatter = formatter + + def write(self, record: dict[str, Any]) -> None: + self._writer.write(self._format(record)) + + def write_many(self, records: list[dict[str, Any]]) -> None: + formatted = [self._format(record) for record in records] + write_many = getattr(self._writer, "write_many", None) + if callable(write_many): + write_many(formatted) + return + for record in formatted: + self._writer.write(record) + + def close(self) -> None: + close = getattr(self._writer, "close", None) + if callable(close): close() - except Exception as exc: - safe_report( - on_failure, - "logging.destination_close", - exc, - destination=getattr(destination, "name", None), - ) + + def _format(self, record: dict[str, Any]) -> dict[str, Any]: + value = deepcopy(record) + return self._formatter(value) if self._formatter else value -def clamped(value: Any, *, low: float, high: float, default: float) -> float: - """Coerce a config knob to a finite float within [low, high]. +@dataclass(frozen=True, slots=True) +class CustomLogDestination: + """Adapts an application or third-party writer into the runtime. - Anything unparseable or non-finite falls back to the default, so a - stray env value can never disable or unbound the mechanism it tunes. + Queued delivery is the safe default. Use inline delivery only for writers + that perform bounded local work and never make network requests. """ - try: - number = float(value) - except (TypeError, ValueError): - return default - if not math.isfinite(number): - return default - return min(max(number, low), high) - - -def normalize_payload(value: Any) -> Any: - if value is None or isinstance(value, str | bool | int | float): - return value - if isinstance(value, Mapping): - return { - str(key): normalize_payload(item) for key, item in value.items() - } - if isinstance(value, Sequence) and not isinstance(value, str | bytes): - return [normalize_payload(item) for item in value] - if isinstance(value, bytes): - try: - return value.decode("utf-8") - except UnicodeDecodeError: - return repr(value) - try: - return str(value) - except BaseException: - return f"" + + name: str + writer_factory: Callable[[], RecordWriter] + delivery: DeliveryMode = "queued" + queue_capacity: int = 1_000 + batch_size: int = 1 + formatter: RecordFormatter | None = None + + def diagnostics(self) -> tuple[str, ...]: + errors: list[str] = [] + if not self.name.strip(): + errors.append("Custom log destination name must be non-empty.") + if not callable(self.writer_factory): + errors.append( + "Custom log destination writer_factory must be callable." + ) + if self.formatter is not None and not callable(self.formatter): + errors.append("Custom log destination formatter must be callable.") + return tuple(errors) + + def build_writer(self, _context: DestinationBuildContext) -> RecordWriter: + return _FormattingWriter(self.writer_factory(), self.formatter) diff --git a/policyengine_observability/destinations/google_cloud.py b/policyengine_observability/destinations/google_cloud.py new file mode 100644 index 0000000..c251eb3 --- /dev/null +++ b/policyengine_observability/destinations/google_cloud.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import functools +from dataclasses import dataclass +from typing import Any, Literal + +from .base import DestinationBuildContext, RecordWriter + +GOOGLE_TRACE_KEY = "logging.googleapis.com/trace" +GOOGLE_SPAN_ID_KEY = "logging.googleapis.com/spanId" +GOOGLE_TRACE_SAMPLED_KEY = "logging.googleapis.com/trace_sampled" + + +@dataclass(frozen=True, slots=True) +class GoogleCloudLogFormatter: + """Adds Cloud Logging trace-correlation fields to a record copy.""" + + project_id: str + + def __call__(self, record: dict[str, Any]) -> dict[str, Any]: + trace_id = record.get("trace_id") + span_id = record.get("span_id") + if trace_id and self.project_id.strip(): + record[GOOGLE_TRACE_KEY] = ( + f"projects/{self.project_id}/traces/{trace_id}" + ) + record[GOOGLE_TRACE_SAMPLED_KEY] = bool( + record.get("trace_sampled", False) + ) + if span_id: + record[GOOGLE_SPAN_ID_KEY] = str(span_id) + return record + + +@dataclass(frozen=True, slots=True) +class GoogleCloudLogDestination: + """Writes records directly to the Google Cloud Logging API.""" + + project_id: str + log_name: str + queue_capacity: int = 1_000 + batch_size: int = 100 + write_timeout_seconds: float = 5.0 + name: str = "google_cloud" + delivery: Literal["queued"] = "queued" + + def build_writer(self, context: DestinationBuildContext) -> RecordWriter: + del context + if not self.project_id.strip() or not self.log_name.strip(): + raise ValueError( + "Google Cloud project_id and log_name must be non-empty." + ) + return _GoogleCloudWriter(self) + + def diagnostics(self) -> tuple[str, ...]: + errors: list[str] = [] + if not self.project_id.strip(): + errors.append("Google Cloud logging requires project_id.") + if not self.log_name.strip(): + errors.append("Google Cloud logging requires log_name.") + _timeout = self.write_timeout_seconds + if ( + not isinstance(_timeout, (int, float)) + or isinstance(_timeout, bool) + or not 0.1 <= _timeout <= 60 + ): + errors.append( + "Google Cloud logging write_timeout_seconds must be between " + "0.1 and 60." + ) + return tuple(errors) + + +class _GoogleCloudWriter: + """Lazy-importing Cloud Logging writer used only by a queue worker.""" + + def __init__(self, config: GoogleCloudLogDestination) -> None: + from google.cloud import logging_v2 + + from ..google_credentials import load_google_credentials + + credentials = load_google_credentials(prefer_workload_identity=True) + self._client = logging_v2.Client( + project=config.project_id, + credentials=credentials, + ) + self._logger = self._client.logger(config.log_name) + self._project_id = config.project_id + self._timeout = max( + 0.1, min(float(config.write_timeout_seconds), 60.0) + ) + self._bound_write_timeout() + logging_v2._instrumentation_emitted = True + + def write(self, record: dict[str, Any]) -> None: + self.write_many([record]) + + def write_many(self, records: list[dict[str, Any]]) -> None: + batch = self._logger.batch() + for record in records: + batch.log_struct(record, **self._entry_kwargs(record)) + batch.commit() + + def _entry_kwargs(self, record: dict[str, Any]) -> dict[str, Any]: + kwargs: dict[str, Any] = { + "severity": record.get("severity", "DEFAULT"), + } + trace_id = record.get("trace_id") + span_id = record.get("span_id") + if trace_id: + kwargs["trace"] = f"projects/{self._project_id}/traces/{trace_id}" + if span_id: + kwargs["span_id"] = str(span_id) + kwargs["trace_sampled"] = bool(record.get("trace_sampled", False)) + return kwargs + + def close(self) -> None: + close = getattr(self._client, "close", None) + if callable(close): + close() + + def _bound_write_timeout(self) -> None: + api = getattr(self._client, "logging_api", None) + gapic = getattr(api, "_gapic_api", None) + if gapic is None: + return + from google.api_core import exceptions as api_exceptions + from google.api_core.retry import Retry, if_exception_type + + retry = Retry( + initial=0.1, + maximum=1.0, + multiplier=1.3, + timeout=self._timeout, + predicate=if_exception_type( + api_exceptions.DeadlineExceeded, + api_exceptions.InternalServerError, + api_exceptions.ServiceUnavailable, + ), + ) + gapic.write_log_entries = functools.partial( + gapic.write_log_entries, + retry=retry, + timeout=self._timeout, + ) diff --git a/policyengine_observability/destinations/google_cloud_logging.py b/policyengine_observability/destinations/google_cloud_logging.py deleted file mode 100644 index 2e1c96e..0000000 --- a/policyengine_observability/destinations/google_cloud_logging.py +++ /dev/null @@ -1,241 +0,0 @@ -from __future__ import annotations - -import functools -from collections.abc import Callable -from datetime import datetime -from typing import Any, Protocol - -from ..config import float_from_env -from .base import clamped, normalize_payload -from .google_credentials import ( - configure_google_application_credentials, - load_google_credentials, -) -from .registry import register_destination -from .stdout import StdoutFormatter, register_stdout_formatter - -DEFAULT_WRITE_TIMEOUT_SECONDS = 10.0 -MIN_WRITE_TIMEOUT_SECONDS = 0.5 -MAX_WRITE_TIMEOUT_SECONDS = 60.0 - -# Structured-JSON keys the Cloud Run/GKE logging agent promotes to -# first-class LogEntry fields when it ingests a stdout line. -GOOGLE_TRACE_KEY = "logging.googleapis.com/trace" -GOOGLE_SPAN_ID_KEY = "logging.googleapis.com/spanId" -GOOGLE_LABELS_KEY = "logging.googleapis.com/labels" - - -class GoogleCloudLogger(Protocol): - def log_struct( - self, - payload: dict[str, Any], - **kwargs: Any, - ) -> None: ... - - -class GoogleCloudLoggingClient(Protocol): - project: str | None - - def logger(self, log_name: str) -> GoogleCloudLogger: ... - - -GoogleCloudLoggingClientFactory = Callable[ - [str | None, object | None], - GoogleCloudLoggingClient, -] - - -class GoogleCloudLoggingDestination: - name = "google_cloud_logging" - - def __init__( - self, - *, - project: str | None, - log_name: str, - client_factory: GoogleCloudLoggingClientFactory | None = None, - write_timeout_seconds: float | None = None, - ) -> None: - self.project = project - self.log_name = log_name - self.write_timeout_seconds = clamped( - write_timeout_seconds, - low=MIN_WRITE_TIMEOUT_SECONDS, - high=MAX_WRITE_TIMEOUT_SECONDS, - default=DEFAULT_WRITE_TIMEOUT_SECONDS, - ) - credentials = load_google_credentials(prefer_workload_identity=True) - if credentials is None: - configure_google_application_credentials() - if client_factory is None: - from google.cloud import logging as cloud_logging - - def client_factory( - project_id: str | None, - credentials: object | None, - ) -> GoogleCloudLoggingClient: - return cloud_logging.Client( - project=project_id, - credentials=credentials, - ) - - self.client = client_factory(project, credentials) - self.project = project or getattr(self.client, "project", None) - self.logger = self.client.logger(log_name) - self._bound_write_timeout() - self._suppress_instrumentation_entry() - - def _suppress_instrumentation_entry(self) -> None: - """Keep the library's diagnostic entry out of the log stream. - - ``log_struct`` prepends a one-time instrumentation diagnostic - entry to the first write per process; observability entries - should be only the records we were asked to write. - """ - try: - from google.cloud import logging_v2 - - logging_v2._instrumentation_emitted = True - except ImportError: # pragma: no cover - google extra has it - pass - - def _bound_write_timeout(self) -> None: - """Bound every write on this destination's own client. - - ``log_struct`` exposes no call options, and the transport default - lets a degraded Logging API hold one write for up to 60 seconds. - The gapic method is the single choke point underneath - ``log_struct``, so rebind it on this client instance with a retry - whose transient-error budget and per-call timeout are both capped - by ``write_timeout_seconds``. When the private handle is absent - (HTTP transport, injected fakes), the library default applies — - acceptable because no write on this destination ever runs on a - request thread. - """ - api = getattr(self.client, "logging_api", None) - gapic = getattr(api, "_gapic_api", None) - if gapic is None: - return - try: - from google.api_core import exceptions as api_exceptions - from google.api_core.retry import Retry, if_exception_type - except ImportError: # pragma: no cover - google extra always has it - return - retry = Retry( - initial=0.1, - maximum=1.0, - multiplier=1.3, - timeout=self.write_timeout_seconds, - predicate=if_exception_type( - api_exceptions.DeadlineExceeded, - api_exceptions.InternalServerError, - api_exceptions.ServiceUnavailable, - ), - ) - gapic.write_log_entries = functools.partial( - gapic.write_log_entries, - retry=retry, - timeout=self.write_timeout_seconds, - ) - - def emit( - self, - payload: dict[str, Any], - *, - log_type: str, - severity: str, - timestamp: datetime | None = None, - ) -> None: - normalized = normalize_payload(payload) - kwargs: dict[str, Any] = { - "severity": severity, - "labels": _labels(normalized, log_type=log_type), - } - if timestamp is not None: - # Supplied by queued transports so delayed writes keep the - # record's event time instead of the delivery time. - kwargs["timestamp"] = timestamp - trace_id = normalized.get("trace_id") - if trace_id and self.project: - kwargs["trace"] = _trace_resource(self.project, trace_id) - span_id = normalized.get("span_id") - if span_id: - kwargs["span_id"] = span_id - self.logger.log_struct(normalized, **kwargs) - - def close(self) -> None: - """Release the owned client's transport, if it supports it.""" - close = getattr(self.client, "close", None) - if callable(close): - close() - - -def _trace_resource(project: str, trace_id: str) -> str: - # The LogEntry trace resource name; the direct write path and the - # agent-native stdout formatter must build it identically for trace - # correlation to work. - return f"projects/{project}/traces/{trace_id}" - - -def _labels(payload: dict[str, Any], *, log_type: str) -> dict[str, str]: - labels = {"log_type": log_type} - for key in ( - "service_name", - "service_role", - "environment", - "schema_version", - ): - value = payload.get(key) - if value is not None: - labels[key] = str(value) - return labels - - -def _google_stdout_formatter_factory(config: Any) -> StdoutFormatter: - """Shape stdout lines with the agent-native Cloud Logging keys. - - Emission stays synchronous, so no ``time`` key is set — the agent's - receive time is the event time. - """ - project = getattr(config, "google_cloud_project", None) - - def format_google( - payload: dict[str, Any], *, log_type: str, severity: str - ) -> dict[str, Any]: - payload["severity"] = str(severity).upper() - payload[GOOGLE_LABELS_KEY] = _labels(payload, log_type=log_type) - trace_id = payload.get("trace_id") - if trace_id and project: - payload[GOOGLE_TRACE_KEY] = _trace_resource(project, trace_id) - span_id = payload.get("span_id") - if span_id: - payload[GOOGLE_SPAN_ID_KEY] = str(span_id) - return payload - - return format_google - - -register_stdout_formatter("google", _google_stdout_formatter_factory) - - -def _google_destination_factory(*, config: Any, **_: Any): - return GoogleCloudLoggingDestination( - project=config.google_cloud_project, - log_name=config.google_cloud_log_name, - # Backend knobs belong to the strategy: parsed here at - # construction (so re-read on restart_observability()), keeping - # per-backend fields off the core config dataclass. - write_timeout_seconds=float_from_env( - "OBSERVABILITY_GOOGLE_WRITE_TIMEOUT_SECONDS", - DEFAULT_WRITE_TIMEOUT_SECONDS, - ), - ) - - -register_destination( - "google_cloud_logging", - _google_destination_factory, - transport="remote", - aliases=("google", "google_cloud"), - required_config=("google_cloud_project",), -) diff --git a/policyengine_observability/destinations/google_credentials.py b/policyengine_observability/destinations/google_credentials.py deleted file mode 100644 index bed34be..0000000 --- a/policyengine_observability/destinations/google_credentials.py +++ /dev/null @@ -1,176 +0,0 @@ -from __future__ import annotations - -import json -import os -import tempfile -from pathlib import Path -from typing import Any - -OIDC_TOKEN_ENV = "OBSERVABILITY_GOOGLE_OIDC_TOKEN" -MODAL_IDENTITY_TOKEN_ENV = "MODAL_IDENTITY_TOKEN" -WORKLOAD_IDENTITY_PROVIDER_ENV = ( - "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER" -) -SERVICE_ACCOUNT_EMAIL_ENV = "OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL" -STS_TOKEN_URL_ENV = "OBSERVABILITY_GOOGLE_STS_TOKEN_URL" -DEFAULT_STS_TOKEN_URL = "https://sts.googleapis.com/v1/token" -JWT_SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:jwt" -GOOGLE_CREDENTIAL_SCOPES = ("https://www.googleapis.com/auth/cloud-platform",) - - -def configure_google_application_credentials( - *, - credentials_json_env: str = "GCP_CREDENTIALS_JSON", - application_credentials_env: str = "GOOGLE_APPLICATION_CREDENTIALS", - credentials_path: Path | None = None, -) -> Path | None: - try: - existing_path = os.getenv(application_credentials_env) - if existing_path: - return Path(existing_path) - - path = _materialize_json_credentials( - credentials_json_env=credentials_json_env, - credentials_path=credentials_path, - ) - if path is None: - path = _materialize_workload_identity_credentials() - if path is None: - return None - - os.environ[application_credentials_env] = str(path) - return path - except Exception: - return None - - -def load_google_credentials( - *, - credentials_json_env: str = "GCP_CREDENTIALS_JSON", - application_credentials_env: str = "GOOGLE_APPLICATION_CREDENTIALS", - credentials_path: Path | None = None, - prefer_workload_identity: bool = False, -) -> Any | None: - try: - path: Path | None = None - if prefer_workload_identity: - path = _materialize_workload_identity_credentials() - if path is None: - existing_path = os.getenv(application_credentials_env) - if existing_path: - path = Path(existing_path) - if path is None: - path = _materialize_json_credentials( - credentials_json_env=credentials_json_env, - credentials_path=credentials_path, - ) - if path is None and not prefer_workload_identity: - path = _materialize_workload_identity_credentials() - if path is None: - return None - - return _load_credentials_from_file(path) - except Exception: - return None - - -def _materialize_json_credentials( - *, - credentials_json_env: str, - credentials_path: Path | None, -) -> Path | None: - credentials_json = os.getenv(credentials_json_env) - if not credentials_json: - return None - - json.loads(credentials_json) - - path = credentials_path or Path(tempfile.gettempdir()).joinpath( - "policyengine-observability-gcp.json" - ) - path.write_text(credentials_json) - path.chmod(0o600) - return path - - -def _materialize_workload_identity_credentials() -> Path | None: - token = os.getenv(OIDC_TOKEN_ENV) or os.getenv(MODAL_IDENTITY_TOKEN_ENV) - provider = os.getenv(WORKLOAD_IDENTITY_PROVIDER_ENV) - if not token or not provider: - return None - - directory = Path(tempfile.gettempdir()) - token_path = directory / "policyengine-observability-oidc.jwt" - config_path = directory / "policyengine-observability-wif.json" - - token_path.write_text(token) - token_path.chmod(0o600) - config = _external_account_config( - provider=provider, - token_path=token_path, - service_account_email=os.getenv(SERVICE_ACCOUNT_EMAIL_ENV), - token_url=os.getenv(STS_TOKEN_URL_ENV) or DEFAULT_STS_TOKEN_URL, - ) - config_path.write_text(json.dumps(config)) - config_path.chmod(0o600) - return config_path - - -def _load_credentials_from_file(path: Path) -> Any: - config = json.loads(path.read_text()) - scopes = list(GOOGLE_CREDENTIAL_SCOPES) - - if config.get("type") == "external_account": - from google.auth import identity_pool - - return identity_pool.Credentials.from_info(config, scopes=scopes) - - if config.get("type") == "service_account": - from google.oauth2.service_account import Credentials - - return Credentials.from_service_account_file( - str(path), - scopes=scopes, - ) - - import google.auth - - credentials, _project = google.auth.load_credentials_from_file( - str(path), - scopes=scopes, - ) - return credentials - - -def _external_account_config( - *, - provider: str, - token_path: Path, - service_account_email: str | None, - token_url: str, -) -> dict[str, object]: - config: dict[str, object] = { - "type": "external_account", - "audience": _workload_identity_audience(provider), - "subject_token_type": JWT_SUBJECT_TOKEN_TYPE, - "token_url": token_url, - "credential_source": { - "file": str(token_path), - "format": {"type": "text"}, - }, - } - if service_account_email: - config["service_account_impersonation_url"] = ( - "https://iamcredentials.googleapis.com/v1/projects/-/" - f"serviceAccounts/{service_account_email}:generateAccessToken" - ) - return config - - -def _workload_identity_audience(provider: str) -> str: - value = provider.strip() - if value.startswith("//iam.googleapis.com/"): - return value - if value.startswith("projects/"): - return f"//iam.googleapis.com/{value}" - return value diff --git a/policyengine_observability/destinations/manager.py b/policyengine_observability/destinations/manager.py deleted file mode 100644 index 086e377..0000000 --- a/policyengine_observability/destinations/manager.py +++ /dev/null @@ -1,223 +0,0 @@ -from __future__ import annotations - -import logging -import time -from collections.abc import Callable, Mapping -from typing import Any - -from ..config import ObservabilityConfig -from .base import LogDestination, close_destination -from .queued import QueuedLogDestination -from .registry import destination_strategy -from .stdout import StdoutJsonDestination, build_stdout_destination - -# A destination that fails this many consecutive emits is disabled for the -# rest of the process. Inline destinations emit synchronously on the -# caller's (request) path, so a persistently failing one — e.g. a broken -# serializer — must not keep charging every request; an observability sink -# can never be allowed to degrade the host service. Queued destinations -# never raise from emit (drops are counted internally), so this breaker -# only ever governs inline strategies. -DESTINATION_FAILURE_LIMIT = 3 - - -class LogDestinationManager: - def __init__( - self, - *, - config: ObservabilityConfig, - loggers: Mapping[str, logging.Logger], - serializer: Callable[[dict[str, Any]], str], - on_failure: Callable[..., None], - ) -> None: - self.config = config - self.loggers = loggers - self.serializer = serializer - self.on_failure = on_failure - self.destinations: list[LogDestination] = [] - self.configured = False - self._consecutive_failures: dict[int, int] = {} - - def configure(self) -> None: - # Reconfigure (restart_observability) runs only from - # single-threaded lifecycle moments by documented contract. - # Construction-time reports are deferred until the new - # destinations are installed: reporting routes through emit, so - # firing mid-build would recurse into configure. - previous = self.destinations - deferred: list[tuple[str, BaseException, dict[str, Any]]] = [] - - def deferred_report( - operation: str, exc: BaseException, **fields: Any - ) -> None: - deferred.append((operation, exc, fields)) - - destinations: list[LogDestination] = [] - for destination_name in self.config.log_destinations or ("stdout",): - try: - destinations.append( - self._build_destination( - destination_name, - build_on_failure=deferred_report, - ) - ) - except BaseException as exc: - deferred_report( - "logging.destination_config", - exc, - destination=destination_name, - ) - if not destinations: - destinations.append(self._stdout_destination()) - deferred_report( - "logging.destination_config", - RuntimeError( - "No configured observability log destination " - "initialized; falling back to stdout." - ), - destination="stdout_fallback", - ) - self.destinations = destinations - # The failure ledger is keyed by id(); clear it with the swap so - # stale entries cannot attach to a new destination via id reuse. - self._consecutive_failures.clear() - self.configured = True - # Close the replaced destinations only after the new ones are - # installed, so their close-phase failure reports (which route - # through emit) still have a sink. - self._close_destinations(previous) - for operation, exc, fields in deferred: - self.on_failure(operation, exc, **fields) - for warning in getattr(self.config, "config_warnings", ()): - self.on_failure("logging.profile_config", ValueError(warning)) - - def emit( - self, - payload: dict[str, Any], - *, - log_type: str, - severity: str, - ) -> None: - emitted_payload = {**payload, "severity": severity} - tripped: list[LogDestination] = [] - for destination in self._ensure_destinations(): - try: - destination.emit( - emitted_payload, - log_type=log_type, - severity=severity, - ) - self._consecutive_failures.pop(id(destination), None) - except BaseException as exc: - failures = ( - self._consecutive_failures.get(id(destination), 0) + 1 - ) - self._consecutive_failures[id(destination)] = failures - if failures >= DESTINATION_FAILURE_LIMIT: - tripped.append(destination) - self.on_failure( - "logging.destination_emit", - exc, - destination=getattr(destination, "name", None), - log_type=log_type, - consecutive_failures=failures, - ) - for destination in tripped: - self._disable_destination(destination) - - def close(self, deadline_seconds: float | None = None) -> None: - """Close every destination that supports closing, best-effort.""" - self._close_destinations(self.destinations, deadline_seconds) - - def _close_destinations( - self, - destinations: list[LogDestination], - deadline_seconds: float | None = None, - ) -> None: - # One deadline covers the whole batch: each close gets whatever - # budget the earlier ones left, so N stuck destinations cannot - # take N times the budget. None means each destination applies - # its own default (a reconfigure, not a bounded shutdown). - deadline = ( - None - if deadline_seconds is None - else time.monotonic() + max(0.0, deadline_seconds) - ) - for destination in destinations: - remaining = ( - None - if deadline is None - else max(0.0, deadline - time.monotonic()) - ) - close_destination( - destination, - on_failure=self.on_failure, - deadline_seconds=remaining, - ) - - def _ensure_destinations(self) -> list[LogDestination]: - if not self.configured: - try: - self.configure() - except BaseException as exc: - self.destinations = [self._stdout_destination()] - self.configured = True - self.on_failure("logging.destination_config", exc) - return self.destinations - - def _disable_destination(self, destination: LogDestination) -> None: - self.destinations = [ - existing - for existing in self.destinations - if existing is not destination - ] - self._consecutive_failures.pop(id(destination), None) - self.on_failure( - "logging.destination_disabled", - RuntimeError( - "Disabling observability log destination after " - f"{DESTINATION_FAILURE_LIMIT} consecutive emit failures." - ), - destination=getattr(destination, "name", None), - ) - if not self.destinations: - self.destinations.append(self._stdout_destination()) - - def _build_destination( - self, - destination_name: str, - *, - build_on_failure: Callable[..., None], - ) -> LogDestination: - strategy = destination_strategy(destination_name) - if strategy is None: - raise ValueError( - f"Unknown observability log destination: {destination_name}" - ) - destination = strategy.factory( - config=self.config, - loggers=self.loggers, - serializer=self.serializer, - on_failure=build_on_failure, - ) - if strategy.transport == "remote": - # No remote strategy may ever write on a request thread. - destination = QueuedLogDestination( - inner=destination, - on_failure=self.on_failure, - maxsize=self.config.log_queue_maxsize, - close_timeout_seconds=( - self.config.log_queue_close_timeout_seconds - ), - ) - return destination - - def _stdout_destination(self) -> StdoutJsonDestination: - # The fail-open fallback: built directly (registry-free) and - # with reporting suppressed, because this can run from inside a - # failure-reporting path where another report would recurse. - return build_stdout_destination( - config=self.config, - loggers=self.loggers, - serializer=self.serializer, - ) diff --git a/policyengine_observability/destinations/queued.py b/policyengine_observability/destinations/queued.py deleted file mode 100644 index 1b5f30f..0000000 --- a/policyengine_observability/destinations/queued.py +++ /dev/null @@ -1,300 +0,0 @@ -"""Bounded, best-effort background delivery for remote log destinations. - -``QueuedLogDestination`` wraps any ``LogDestination`` so the caller's -thread only ever enqueues: a stdlib ``logging.handlers.QueueListener`` -thread performs the actual writes, so a degraded sink can never stall a -request. Delivery is best-effort by contract — the stdout sibling -destination is the durable record — so a full queue drops the newest -record and counts it, and there is deliberately no circuit breaker, -retry queue, or recovery machinery in this component. - -Mutable state census (any addition needs design review): - -1. ``_queue`` — thread-safe by construction (``queue.Queue``). -2. ``_listener``— started once in ``__init__``, stopped once in ``close``. -3. ``_drops`` — best-effort drop counter with its throttle state. -4. ``_closed`` — one-way flag flipped by ``close``. - -(The handler's write-failure counter is confined to the listener -thread, so it is not shared mutable state.) - -Accepted races, all bounded and within the best-effort contract: - -- An ``emit`` that passes the ``_closed`` check while ``close`` runs can - lose that one record uncounted. -- When ``close`` times out, the daemon listener thread is abandoned; it - keeps draining in the background until process exit. -- A process that forks after construction (e.g. gunicorn ``--preload``) - inherits a dead listener; records drop with ``reason="full"`` until - the child calls ``restart_observability()`` from a post-fork hook. - There are deliberately no fork hooks or pid checks here. -""" - -from __future__ import annotations - -import atexit -import queue as queue_module -import time -from collections.abc import Callable -from dataclasses import dataclass -from datetime import UTC, datetime -from logging.handlers import QueueListener -from typing import Any - -from ..config import ( - DEFAULT_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS, - DEFAULT_LOG_QUEUE_MAXSIZE, -) -from .base import ( - LogDestination, - accepts_keyword, - clamped, - close_destination, - normalize_payload, - safe_report, -) - -MIN_QUEUE_MAXSIZE = 10 -MAX_QUEUE_MAXSIZE = 100_000 -MIN_CLOSE_TIMEOUT_SECONDS = 0.0 -MAX_CLOSE_TIMEOUT_SECONDS = 30.0 -DROP_REPORT_INTERVAL = 100 -WRITE_FAILURE_REPORT_INTERVAL = 100 - - -@dataclass(frozen=True, slots=True) -class _QueuedRecord: - payload: Any - log_type: str - severity: str - enqueued_at: datetime - - -class _ThrottledCounter: - """Count occurrences; say when one should be reported. - - The shared throttle policy for failure-path reporting: the first - occurrence always reports, then every ``interval``-th, so a - persistent problem stays visible without flooding the internal-error - channel. - """ - - __slots__ = ("count", "interval") - - def __init__(self, interval: int) -> None: - self.interval = max(1, int(interval)) - self.count = 0 - - def tick(self) -> int | None: - """Increment; return the count when this occurrence reports.""" - self.count += 1 - if self.count == 1 or self.count % self.interval == 0: - return self.count - return None - - -class _QueuedRecordHandler: - """Duck-typed QueueListener handler: only ``handle`` is ever called. - - The failure counter is confined to the listener thread. A write - failure must never kill the listener, so everything below the emit - is guarded; ``BaseException`` is deliberately not caught (swallowing - ``SystemExit`` on a worker thread is worse than losing the queue — - the consequences of a dead listener are bounded to counted drops). - """ - - def __init__( - self, - inner: LogDestination, - on_failure: Callable[..., None], - *, - forward_timestamp: bool, - report_interval: int = WRITE_FAILURE_REPORT_INTERVAL, - ) -> None: - self.inner = inner - self.on_failure = on_failure - self.forward_timestamp = forward_timestamp - self.failures = _ThrottledCounter(report_interval) - - def handle(self, record: _QueuedRecord) -> None: - try: - if self.forward_timestamp: - self.inner.emit( - record.payload, - log_type=record.log_type, - severity=record.severity, - timestamp=record.enqueued_at, - ) - else: - self.inner.emit( - record.payload, - log_type=record.log_type, - severity=record.severity, - ) - except Exception as exc: - count = self.failures.tick() - if count is None: - return - safe_report( - self.on_failure, - "logging.queue_write", - exc, - destination=getattr(self.inner, "name", None), - log_type=record.log_type, - write_failures_total=count, - ) - - -class _BoundedQueueListener(QueueListener): - """QueueListener whose stop can be given a hard deadline. - - The stdlib ``stop()`` enqueues its sentinel with ``put_nowait`` - (which raises on a jammed bounded queue) and then joins the worker - thread without a timeout. Here one monotonic deadline covers both - the blocking sentinel put and the join; on expiry the daemon thread - is abandoned and ``False`` is returned. - """ - - def stop(self, timeout: float | None = None) -> bool: - thread = self._thread - if thread is None: - return True - if timeout is None: - super().stop() - return True - deadline = time.monotonic() + max(0.0, timeout) - try: - self.queue.put( - self._sentinel, - timeout=max(0.0, deadline - time.monotonic()), - ) - except queue_module.Full: - pass - thread.join(max(0.0, deadline - time.monotonic())) - stopped = not thread.is_alive() - self._thread = None - return stopped - - -class QueuedLogDestination: - def __init__( - self, - *, - inner: LogDestination, - on_failure: Callable[..., None], - maxsize: float = DEFAULT_LOG_QUEUE_MAXSIZE, - close_timeout_seconds: float = DEFAULT_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS, - drop_report_interval: int = DROP_REPORT_INTERVAL, - ) -> None: - self.inner = inner - self.on_failure = on_failure - self.name = f"queued_{getattr(inner, 'name', 'destination')}" - self.maxsize = int( - clamped( - maxsize, - low=MIN_QUEUE_MAXSIZE, - high=MAX_QUEUE_MAXSIZE, - default=DEFAULT_LOG_QUEUE_MAXSIZE, - ) - ) - self.close_timeout_seconds = clamped( - close_timeout_seconds, - low=MIN_CLOSE_TIMEOUT_SECONDS, - high=MAX_CLOSE_TIMEOUT_SECONDS, - default=DEFAULT_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS, - ) - self._queue: queue_module.Queue[_QueuedRecord | None] = ( - queue_module.Queue(self.maxsize) - ) - self._listener = _BoundedQueueListener( - self._queue, - _QueuedRecordHandler( - inner, - on_failure, - forward_timestamp=accepts_keyword(inner.emit, "timestamp"), - ), - ) - self._drops = _ThrottledCounter(drop_report_interval) - self._closed = False - # Construction happens at configure time on the startup thread, - # never lazily on a request thread. - self._listener.start() - atexit.register(self.close) - - def emit( - self, - payload: dict[str, Any], - *, - log_type: str, - severity: str, - ) -> None: - try: - if self._closed: - self._record_drop("closed", log_type) - return - record = _QueuedRecord( - # Snapshot now: callers keep mutating nested structures - # after emit returns, and the write happens later on the - # listener thread. The enqueue time becomes the entry - # timestamp so delayed writes keep event time. - payload=normalize_payload(payload), - log_type=log_type, - severity=severity, - enqueued_at=datetime.now(UTC), - ) - try: - self._queue.put_nowait(record) - except queue_module.Full: - self._record_drop("full", log_type) - except Exception as exc: - self._record_drop("exception", log_type, exc=exc) - - def close(self, deadline_seconds: float | None = None) -> None: - if self._closed: - return - self._closed = True - atexit.unregister(self.close) - deadline = clamped( - deadline_seconds, - low=MIN_CLOSE_TIMEOUT_SECONDS, - high=MAX_CLOSE_TIMEOUT_SECONDS, - default=self.close_timeout_seconds, - ) - drained = self._listener.stop(timeout=deadline) - if not drained: - safe_report( - self.on_failure, - "logging.queue_close_timeout", - TimeoutError( - "Observability log queue did not drain before " - "the close deadline; remaining records are lost." - ), - destination=self.name, - deadline_seconds=deadline, - pending_records=self._queue.qsize(), - ) - # The abandoned listener may still be mid-write; leave the - # inner destination alone rather than closing it underneath - # an active write. - return - close_destination(self.inner, on_failure=self.on_failure) - - def _record_drop( - self, - reason: str, - log_type: str, - exc: BaseException | None = None, - ) -> None: - count = self._drops.tick() - if count is None: - return - safe_report( - self.on_failure, - "logging.queue_drop", - exc or RuntimeError("Observability log queue dropped a record."), - destination=self.name, - log_type=log_type, - reason=reason, - dropped_total=count, - queue_maxsize=self.maxsize, - ) diff --git a/policyengine_observability/destinations/registry.py b/policyengine_observability/destinations/registry.py deleted file mode 100644 index 25b9521..0000000 --- a/policyengine_observability/destinations/registry.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Named destination strategies. - -Backend modules register themselves here so the manager can build -destinations without knowing any backend: an ``inline`` strategy writes -synchronously on the caller's thread (stdout — the durable, dependency- -free record), while every ``remote`` strategy is wrapped in the bounded -queue transport, so no remote write can ever run on a request thread. -""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from typing import Literal - -from .base import LogDestination, normalize_name - -# Factories are called with keyword arguments (config, loggers, -# serializer, on_failure) and may ignore what they do not need. The -# on_failure callable is for construction-time reporting only; reports -# made through it may be deferred until the build completes. -DestinationFactory = Callable[..., LogDestination] - - -@dataclass(frozen=True) -class DestinationStrategy: - factory: DestinationFactory - transport: Literal["inline", "remote"] - # Config attribute names that must resolve truthy for the strategy - # to be usable. Profiles naming this strategy downgrade gracefully - # (with a warning) when a requirement is missing, instead of failing - # at construction and falling back with a config-failure report. - required_config: tuple[str, ...] = () - - -_STRATEGIES: dict[str, DestinationStrategy] = {} - - -def register_destination( - name: str, - factory: DestinationFactory, - *, - transport: Literal["inline", "remote"], - aliases: tuple[str, ...] = (), - required_config: tuple[str, ...] = (), -) -> None: - strategy = DestinationStrategy( - factory=factory, - transport=transport, - required_config=required_config, - ) - for key in (name, *aliases): - _STRATEGIES[normalize_name(key)] = strategy - - -def destination_strategy(name: str) -> DestinationStrategy | None: - return _STRATEGIES.get(normalize_name(name)) diff --git a/policyengine_observability/destinations/stdout.py b/policyengine_observability/destinations/stdout.py index 2f562f5..e219c67 100644 --- a/policyengine_observability/destinations/stdout.py +++ b/policyengine_observability/destinations/stdout.py @@ -1,138 +1,56 @@ from __future__ import annotations -import logging -from collections.abc import Callable, Mapping -from typing import Any +import json +from copy import deepcopy +from dataclasses import dataclass +from typing import Any, Literal -from .base import normalize_name, normalize_payload, safe_report -from .registry import register_destination +from .base import ( + DestinationBuildContext, + RecordFormatter, + RecordWriter, +) -# A stdout formatter shapes the normalized payload into the JSON line a -# platform's log agent expects. Formatters are registered by name so -# backend modules can contribute agent-native shapes without the core -# knowing about any backend; "plain" is the built-in default. Factories -# receive the ObservabilityConfig so a formatter can close over settings -# it needs (duck-typed to keep this module config-agnostic). -StdoutFormatter = Callable[..., dict[str, Any]] -StdoutFormatterFactory = Callable[[Any], StdoutFormatter] - -_FORMATTER_FACTORIES: dict[str, StdoutFormatterFactory] = {} - - -def register_stdout_formatter( - name: str, factory: StdoutFormatterFactory -) -> None: - _FORMATTER_FACTORIES[normalize_name(name)] = factory - - -def resolve_stdout_formatter( - config: Any, - on_failure: Callable[..., None] | None = None, -) -> StdoutFormatter: - raw = getattr(config, "stdout_format", None) or "plain" - factory = _FORMATTER_FACTORIES.get(normalize_name(raw)) - if factory is None: - # Falling back must not lose the record, but a typo'd format - # name should not pass silently either — the agent-native shape - # it named would just quietly never appear. - if on_failure is not None: - safe_report( - on_failure, - "logging.stdout_format", - ValueError(f"Unknown stdout format {raw!r}; using plain."), - ) - factory = _FORMATTER_FACTORIES["plain"] - try: - return factory(config) - except Exception as exc: - # Stdout is the fail-open record, and this resolver also runs on - # the manager's last-resort fallback path: a registered factory - # that raises must degrade to the built-in plain formatter (not - # the registry entry, which could be the broken one), never - # break configure or startup. - if on_failure is not None: - safe_report( - on_failure, - "logging.stdout_format", - exc, - stdout_format=raw, - ) - return _plain_formatter_factory(config) - - -def _plain_formatter_factory(config: Any) -> StdoutFormatter: - def format_plain( - payload: dict[str, Any], *, log_type: str, severity: str - ) -> dict[str, Any]: - return payload - - return format_plain - - -register_stdout_formatter("plain", _plain_formatter_factory) - - -class StdoutJsonDestination: - name = "stdout" +class _StdoutWriter: def __init__( self, - *, - loggers: Mapping[str, logging.Logger], - serializer: Callable[[dict[str, Any]], str], - formatter: StdoutFormatter | None = None, - ) -> None: - self.loggers = loggers - self.serializer = serializer - self.formatter = formatter or _plain_formatter_factory(None) - - def emit( - self, - payload: dict[str, Any], - *, - log_type: str, - severity: str, + context: DestinationBuildContext, + formatter: RecordFormatter | None, ) -> None: - normalized = normalize_payload(payload) - try: - # Formatters receive (and may mutate) the private normalized - # copy. Stdout is the fallback sink, so a broken formatter - # degrades to the unformatted line rather than losing the - # record or tripping the breaker. - formatted = self.formatter( - normalized, log_type=log_type, severity=severity - ) - except Exception: - formatted = normalized - message = self.serializer(formatted) - logger = self.loggers.get(log_type) or self.loggers["event"] - if severity in {"ERROR", "CRITICAL"}: - logger.error(message) - elif severity == "WARNING": - logger.warning(message) - else: - logger.info(message) - - -def build_stdout_destination( - *, - config: Any, - loggers: Any, - serializer: Any, - on_failure: Callable[..., None] | None = None, - **_: Any, -) -> StdoutJsonDestination: - """The one place a configured stdout destination is assembled. - - Used both as the registered ``stdout`` strategy factory and by the - manager's fail-open fallback, so formatter resolution can never - diverge between the two paths. - """ - return StdoutJsonDestination( - loggers=loggers, - serializer=serializer, - formatter=resolve_stdout_formatter(config, on_failure=on_failure), - ) - - -register_destination("stdout", build_stdout_destination, transport="inline") + self._context = context + self._formatter = formatter + + def write(self, record: dict[str, Any]) -> None: + value = deepcopy(record) + if self._formatter is not None: + value = self._formatter(value) + print( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ), + file=self._context.stdout(), + flush=True, + ) + + +@dataclass(frozen=True, slots=True) +class StdoutLogDestination: + """Writes one JSON object per line to the process standard output.""" + + formatter: RecordFormatter | None = None + name: str = "stdout" + delivery: Literal["inline"] = "inline" + queue_capacity: int = 1 + batch_size: int = 1 + + def diagnostics(self) -> tuple[str, ...]: + if self.formatter is not None and not callable(self.formatter): + return ("Standard-output formatter must be callable.",) + return () + + def build_writer(self, context: DestinationBuildContext) -> RecordWriter: + return _StdoutWriter(context, self.formatter) diff --git a/policyengine_observability/diagnostics.py b/policyengine_observability/diagnostics.py new file mode 100644 index 0000000..79050d8 --- /dev/null +++ b/policyengine_observability/diagnostics.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import json +import sys +import threading +from collections import Counter +from collections.abc import Callable +from datetime import UTC, datetime +from time import monotonic +from typing import Any + + +class Diagnostics: + """Rate-limited local reporting that never enters remote delivery.""" + + def __init__( + self, + *, + interval_seconds: float = 60.0, + stderr: Any = None, + sensitive_values: tuple[str, ...] = (), + ) -> None: + self._interval_seconds = max(1.0, interval_seconds) + self._stderr = stderr or sys.stderr + self._sensitive_values = sensitive_values + self._last_report: dict[str, float] = {} + self._counts: Counter[str] = Counter() + self._listeners: list[Callable[[str, int], None]] = [] + self._lock = threading.Lock() + + def increment(self, name: str, value: int = 1) -> None: + with self._lock: + self._counts[name] += value + listeners = tuple(self._listeners) + for listener in listeners: + try: + listener(name, value) + except Exception: + continue + + def add_listener(self, listener: Callable[[str, int], None]) -> None: + with self._lock: + self._listeners.append(listener) + + def count(self, name: str) -> int: + with self._lock: + return self._counts[name] + + def snapshot(self) -> dict[str, int]: + with self._lock: + return dict(self._counts) + + def restart_after_process_duplication(self) -> None: + """Replace synchronization state inherited across process copying.""" + + self._lock = threading.Lock() + self._last_report = {} + self._counts = Counter() + + def report( + self, + operation: str, + error: Exception | str, + **fields: Any, + ) -> None: + self.increment(f"failure.{operation}") + now = monotonic() + with self._lock: + previous = self._last_report.get(operation) + if ( + previous is not None + and now - previous < self._interval_seconds + ): + return + self._last_report[operation] = now + + record = { + "schema_version": "policyengine.observability.internal.v1", + "timestamp": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + "severity": "ERROR", + "event.name": "observability.internal_failure", + "operation": operation, + "error.type": type(error).__name__, + "error.message": _safe_text(error, self._sensitive_values), + **{ + str(key): _safe_scalar(value, self._sensitive_values) + for key, value in fields.items() + }, + } + try: + print( + json.dumps(record, sort_keys=True, ensure_ascii=True), + file=self._stderr, + flush=True, + ) + except Exception: + return + + +def _safe_text(value: Any, sensitive_values: tuple[str, ...]) -> str: + try: + text = str(value) + except Exception: + return "" + for sensitive in sensitive_values: + if sensitive: + text = text.replace(sensitive, "[REDACTED]") + return text[:2_048] + + +def _safe_scalar( + value: Any, sensitive_values: tuple[str, ...] +) -> str | int | float | bool | None: + if isinstance(value, str): + return _safe_text(value, sensitive_values) + if value is None or isinstance(value, (int, float, bool)): + return value + return _safe_text(value, sensitive_values) diff --git a/policyengine_observability/google_auth.py b/policyengine_observability/google_auth.py new file mode 100644 index 0000000..0f5003e --- /dev/null +++ b/policyengine_observability/google_auth.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any +from urllib.parse import urlparse + +from .config import OTLPProtocol +from .google_credentials import ( + DEFAULT_STS_TOKEN_URL, + GOOGLE_CREDENTIAL_SCOPES, + JWT_SUBJECT_TOKEN_TYPE, + MODAL_IDENTITY_TOKEN_ENV, + OIDC_TOKEN_ENV, + SERVICE_ACCOUNT_EMAIL_ENV, + WORKLOAD_IDENTITY_PROVIDER_ENV, + _EnvironmentSubjectTokenSupplier, + _workload_identity_audience, +) + + +@dataclass(frozen=True, slots=True) +class GoogleIdTokenAuth: + """Authenticates an OTLP exporter to an ID-token protected endpoint.""" + + audience: str + + def exporter_kwargs( + self, + *, + protocol: OTLPProtocol, + headers: dict[str, str], + ) -> dict[str, Any]: + if protocol == "grpc": + return {"credentials": _google_grpc_credentials(self.audience)} + return {"session": _google_http_session(self.audience, headers)} + + +def _google_id_token_credentials(audience: str) -> Any: + from google.auth.transport.requests import Request + + token_env_names = (MODAL_IDENTITY_TOKEN_ENV, OIDC_TOKEN_ENV) + has_subject_token = any(os.getenv(name) for name in token_env_names) + provider = os.getenv(WORKLOAD_IDENTITY_PROVIDER_ENV) + service_account = os.getenv(SERVICE_ACCOUNT_EMAIL_ENV) + if has_subject_token and provider and service_account: + from google.auth import identity_pool, impersonated_credentials + + source = identity_pool.Credentials( + audience=_workload_identity_audience(provider), + subject_token_type=JWT_SUBJECT_TOKEN_TYPE, + token_url=DEFAULT_STS_TOKEN_URL, + subject_token_supplier=_EnvironmentSubjectTokenSupplier( + token_env_names + ), + scopes=list(GOOGLE_CREDENTIAL_SCOPES), + ) + target = impersonated_credentials.Credentials( + source_credentials=source, + target_principal=service_account, + target_scopes=list(GOOGLE_CREDENTIAL_SCOPES), + ) + return impersonated_credentials.IDTokenCredentials( + target_credentials=target, + target_audience=audience, + include_email=True, + ) + + from google.oauth2.id_token import fetch_id_token_credentials + + return fetch_id_token_credentials(audience, request=Request()) + + +def _google_grpc_credentials(audience: str) -> Any: + import grpc + from google.auth.transport.grpc import AuthMetadataPlugin + from google.auth.transport.requests import Request + + plugin = AuthMetadataPlugin( + _google_id_token_credentials(audience), + Request(), + default_host=urlparse(audience).netloc, + ) + return grpc.composite_channel_credentials( + grpc.ssl_channel_credentials(), + grpc.metadata_call_credentials(plugin), + ) + + +def _google_http_session(audience: str, headers: dict[str, str]) -> Any: + from google.auth.transport.requests import AuthorizedSession + + session = AuthorizedSession(_google_id_token_credentials(audience)) + session.headers.update(headers) + return session diff --git a/policyengine_observability/google_credentials.py b/policyengine_observability/google_credentials.py new file mode 100644 index 0000000..26ea344 --- /dev/null +++ b/policyengine_observability/google_credentials.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +OIDC_TOKEN_ENV = "OBSERVABILITY_GOOGLE_OIDC_TOKEN" +MODAL_IDENTITY_TOKEN_ENV = "MODAL_IDENTITY_TOKEN" +WORKLOAD_IDENTITY_PROVIDER_ENV = ( + "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER" +) +SERVICE_ACCOUNT_EMAIL_ENV = "OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL" +STS_TOKEN_URL_ENV = "OBSERVABILITY_GOOGLE_STS_TOKEN_URL" +DEFAULT_STS_TOKEN_URL = "https://sts.googleapis.com/v1/token" +JWT_SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:jwt" +GOOGLE_CREDENTIAL_SCOPES = ("https://www.googleapis.com/auth/cloud-platform",) + + +@dataclass(frozen=True, slots=True) +class _EnvironmentSubjectTokenSupplier: + env_names: tuple[str, ...] + + def get_subject_token(self, _context: Any, _request: Any) -> str: + for name in self.env_names: + token = os.getenv(name) + if token: + return token + from google.auth.exceptions import RefreshError + + names = ", ".join(self.env_names) + raise RefreshError(f"No workload identity token found in {names}.") + + +def load_google_credentials( + *, + credentials_json_env: str = "GCP_CREDENTIALS_JSON", + application_credentials_env: str = "GOOGLE_APPLICATION_CREDENTIALS", + prefer_workload_identity: bool = False, +) -> Any | None: + """Load Google credentials without performing a network request.""" + + try: + if prefer_workload_identity: + credentials = _workload_identity_credentials() + if credentials is not None: + return credentials + + configured_path = os.getenv(application_credentials_env) + if configured_path: + return _load_credentials_from_file(Path(configured_path)) + + credentials_json = os.getenv(credentials_json_env) + if credentials_json: + return _load_credentials_from_json(credentials_json) + + if not prefer_workload_identity: + return _workload_identity_credentials() + return None + except Exception: + return None + + +def _load_credentials_from_json(credentials_json: str) -> Any: + import google.auth + + config = json.loads(credentials_json) + credentials, _project = google.auth.load_credentials_from_dict( + config, + scopes=list(GOOGLE_CREDENTIAL_SCOPES), + ) + return credentials + + +def _workload_identity_credentials() -> Any | None: + token_env_names = (OIDC_TOKEN_ENV, MODAL_IDENTITY_TOKEN_ENV) + if not any(os.getenv(name) for name in token_env_names): + return None + provider = os.getenv(WORKLOAD_IDENTITY_PROVIDER_ENV) + if not provider: + return None + + from google.auth import identity_pool + + kwargs: dict[str, Any] = { + "audience": _workload_identity_audience(provider), + "subject_token_type": JWT_SUBJECT_TOKEN_TYPE, + "token_url": os.getenv(STS_TOKEN_URL_ENV) or DEFAULT_STS_TOKEN_URL, + "subject_token_supplier": _EnvironmentSubjectTokenSupplier( + token_env_names + ), + "scopes": list(GOOGLE_CREDENTIAL_SCOPES), + } + service_account_email = os.getenv(SERVICE_ACCOUNT_EMAIL_ENV) + if service_account_email: + kwargs["service_account_impersonation_url"] = ( + "https://iamcredentials.googleapis.com/v1/projects/-/" + f"serviceAccounts/{service_account_email}:generateAccessToken" + ) + return identity_pool.Credentials(**kwargs) + + +def _load_credentials_from_file(path: Path) -> Any: + config = json.loads(path.read_text()) + scopes = list(GOOGLE_CREDENTIAL_SCOPES) + if config.get("type") == "external_account": + from google.auth import identity_pool + + return identity_pool.Credentials.from_info(config, scopes=scopes) + if config.get("type") == "service_account": + from google.oauth2.service_account import Credentials + + return Credentials.from_service_account_file(str(path), scopes=scopes) + + import google.auth + + credentials, _project = google.auth.load_credentials_from_file( + str(path), scopes=scopes + ) + return credentials + + +def _workload_identity_audience(provider: str) -> str: + value = provider.strip() + if value.startswith("//iam.googleapis.com/"): + return value + if value.startswith("projects/"): + return f"//iam.googleapis.com/{value}" + return value diff --git a/policyengine_observability/integrations/__init__.py b/policyengine_observability/integrations/__init__.py index cbddb89..e2ec12a 100644 --- a/policyengine_observability/integrations/__init__.py +++ b/policyengine_observability/integrations/__init__.py @@ -1 +1,5 @@ """Optional instrumentation integrations.""" + +from .httpx import instrument_httpx + +__all__ = ["instrument_httpx"] diff --git a/policyengine_observability/integrations/httpx.py b/policyengine_observability/integrations/httpx.py index 696d1cd..6a71663 100644 --- a/policyengine_observability/integrations/httpx.py +++ b/policyengine_observability/integrations/httpx.py @@ -1,8 +1,55 @@ from __future__ import annotations -from ..runtime import ObservabilityRuntime, observability_runtime +from typing import Any +from ..runtime import ObservabilityRuntime -def instrument_httpx(runtime: ObservabilityRuntime | None = None) -> None: - runtime = runtime or observability_runtime() - runtime.instrument_httpx() +_RUNTIME_ATTRIBUTE = "_policyengine_observability_runtime" + + +def instrument_httpx(client: Any, runtime: ObservabilityRuntime) -> Any: + """Add correlation headers to requests from one supplied HTTPX client.""" + + existing = getattr(client, _RUNTIME_ATTRIBUTE, None) + if existing is runtime: + return client + if existing is not None: + runtime.diagnostics.report( + "httpx.already_instrumented", + "The HTTPX client is already associated with another runtime.", + ) + return client + + try: + import httpx + + if isinstance(client, httpx.AsyncClient): + + async def inject_async(request: httpx.Request) -> None: + _inject(request, runtime) + + hook = inject_async + elif isinstance(client, httpx.Client): + + def inject_sync(request: httpx.Request) -> None: + _inject(request, runtime) + + hook = inject_sync + else: + raise TypeError("client must be httpx.Client or httpx.AsyncClient") + + client.event_hooks.setdefault("request", []).append(hook) + setattr(client, _RUNTIME_ATTRIBUTE, runtime) + except Exception as exc: + runtime.diagnostics.report("httpx.instrument", exc) + return client + + +def _inject(request: Any, runtime: ObservabilityRuntime) -> None: + try: + headers: dict[str, str] = {} + runtime.inject_http_headers(headers) + for key, value in headers.items(): + request.headers[key] = value + except Exception as exc: + runtime.diagnostics.report("httpx.request_inject", exc) diff --git a/policyengine_observability/logging.py b/policyengine_observability/logging.py deleted file mode 100644 index 666d4ba..0000000 --- a/policyengine_observability/logging.py +++ /dev/null @@ -1,380 +0,0 @@ -"""Structured log emission and observability-failure handling.""" - -from __future__ import annotations - -import json -import logging -import sys -import traceback -from datetime import UTC, datetime -from typing import TYPE_CHECKING, Any - -from .context import ( - ErrorRecord, - OperationObservabilityContext, - RequestObservabilityContext, - _metric_attrs, -) - -if TYPE_CHECKING: - from .runtime import ObservabilityRuntime - - -class PlainMessageFormatter(logging.Formatter): - def format(self, record: logging.LogRecord) -> str: - return record.getMessage() - - -def configure_plain_logger(logger: logging.Logger, level: int) -> None: - logger.setLevel(level) - logger.propagate = False - if not logger.handlers: - handler = logging.StreamHandler() - handler.setFormatter(PlainMessageFormatter()) - logger.addHandler(handler) - - -REQUEST_LOGGER_NAME = "policyengine_observability.requests" -OPERATION_LOGGER_NAME = "policyengine_observability.operations" -EVENT_LOGGER_NAME = "policyengine_observability.events" -INTERNAL_LOGGER_NAME = "policyengine_observability.internal" - -REQUEST_LOGGER = logging.getLogger(REQUEST_LOGGER_NAME) -OPERATION_LOGGER = logging.getLogger(OPERATION_LOGGER_NAME) -EVENT_LOGGER = logging.getLogger(EVENT_LOGGER_NAME) -INTERNAL_LOGGER = logging.getLogger(INTERNAL_LOGGER_NAME) - - -class LogEmitter: - def __init__(self, runtime: ObservabilityRuntime) -> None: - self.runtime = runtime - - def record_error( - self, - exc: BaseException, - *, - handled: bool, - status_code: int | None = None, - include_stack: bool = True, - ) -> None: - if not self.runtime.enabled: - return - try: - context = self.runtime.current_context() - operation = self.runtime.current_operation() - error_record = ErrorRecord( - type=type(exc).__name__, - message=self.runtime._safe_str(exc), - handled=handled, - stack=( - self.runtime._safe_traceback(exc) - if include_stack - else None - ), - ) - if context is not None: - if status_code is not None: - context.status_code = status_code - context.error = error_record - self.runtime.record_error_metric( - context.metric_attributes(error_type=type(exc).__name__) - ) - elif operation is not None: - operation.error = error_record - self.runtime.record_error_metric( - operation.metric_attributes(error_type=type(exc).__name__) - ) - else: - return - span = self.runtime._current_span() - if span is not None: - self.runtime._record_exception_on_span( - span, - exc, - handled=handled, - status_code=status_code, - ) - except BaseException as observability_exc: - self.runtime.log_observability_failure( - "request.record_error", - observability_exc, - original_error_type=type(exc).__name__, - ) - - def record_event(self, event: str, **fields: Any) -> None: - if not self.runtime.enabled: - return - try: - context = self.runtime.current_context() - operation = self.runtime.current_operation() - base: dict[str, Any] = { - "schema_version": "policyengine.observability.event.v1", - "event": event, - "service_name": self.runtime.config.service_name, - "service_role": self.runtime.config.service_role, - "environment": self.runtime.config.environment, - "created_at": datetime.now(UTC).isoformat(), - } - if context is not None: - trace_id, span_id = self.runtime._trace_ids() - base.update( - { - "service_name": context.config.service_name, - "service_role": context.config.service_role, - "environment": context.config.environment, - "request_id": context.request_id, - "trace_id": trace_id, - "span_id": span_id, - "route": context.route, - "path": context.path, - } - ) - elif operation is not None: - trace_id, span_id = self.runtime._trace_ids() - base.update( - { - "service_name": operation.config.service_name, - "service_role": operation.config.service_role, - "environment": operation.config.environment, - "operation": operation.name, - "flavor": operation.flavor, - "trace_id": trace_id, - "span_id": span_id, - } - ) - clean_fields = { - key: value - for key, value in fields.items() - if value is not None - } - base.update(clean_fields) - self.runtime._emit_structured_log( - base, - log_type="event", - severity="INFO", - ) - self.runtime._add_span_event(event, clean_fields) - if event.startswith("modal_") or "fallback" in event: - attrs = ( - context.metric_attributes(event=event) - if context - else operation.metric_attributes(event=event) - if operation - else _metric_attrs( - {"event": event}, - self.runtime.config.metric_attribute_keys, - ) - ) - self.runtime.record_failover_event_metric(attrs) - except BaseException as exc: - self.runtime.log_observability_failure( - "request.record_event", - exc, - event_name=event, - ) - - def emit_request_log(self, context: RequestObservabilityContext) -> None: - if not self.runtime.enabled: - return - try: - if context.emitted: - return - context.emitted = True - if ( - context.internal_dispatch - or not context.config.request_logs_enabled - ): - return - trace_id, span_id = self.runtime._trace_ids() - payload = context.as_log_record( - trace_id=trace_id, - span_id=span_id, - ) - self.runtime._emit_structured_log( - payload, - log_type="request", - severity=self.runtime._severity_for_log_record(payload), - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "request.emit_request_log", - exc, - request_id=getattr(context, "request_id", None), - ) - - def emit_operation_log( - self, - operation: OperationObservabilityContext, - ) -> None: - if not self.runtime.enabled: - return - try: - if operation.emitted: - return - operation.emitted = True - trace_id, span_id = self.runtime._trace_ids() - payload = operation.as_log_record( - trace_id=trace_id, - span_id=span_id, - ) - self.runtime._emit_structured_log( - payload, - log_type="operation", - severity=self.runtime._severity_for_log_record(payload), - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "operation.emit_log", - exc, - operation=getattr(operation, "name", None), - ) - - def log_observability_failure( - self, - operation: str, - exc: BaseException, - **fields: Any, - ) -> None: - payload = self.runtime._internal_error_payload( - operation, exc, **fields - ) - if self.runtime._emitting_internal_error: - self.runtime._write_stderr(payload) - return - self.runtime._emitting_internal_error = True - try: - self.runtime._emit_structured_log( - payload, - log_type="internal", - severity="ERROR", - ) - except BaseException: - self.runtime._write_stderr(payload) - finally: - self.runtime._emitting_internal_error = False - - def _configure_loggers(self) -> None: - for logger in ( - REQUEST_LOGGER, - OPERATION_LOGGER, - EVENT_LOGGER, - INTERNAL_LOGGER, - ): - configure_plain_logger(logger, self.runtime.config.log_level) - - def _emit_structured_log( - self, - payload: dict[str, Any], - *, - log_type: str, - severity: str, - ) -> None: - try: - if not self.runtime.log_destination_manager.configured: - self.runtime._configure_loggers() - self.runtime.log_destination_manager.emit( - payload, - log_type=log_type, - severity=severity, - ) - except BaseException as exc: - if self.runtime._emitting_internal_error: - self.runtime._write_stderr(payload) - else: - self.runtime.log_observability_failure( - "logging.emit", - exc, - log_type=log_type, - ) - - def _handle_destination_failure( - self, - operation: str, - exc: BaseException, - **fields: Any, - ) -> None: - if self.runtime._emitting_internal_error: - self.runtime._write_stderr( - self.runtime._internal_error_payload(operation, exc, **fields) - ) - return - self.runtime.log_observability_failure(operation, exc, **fields) - - def _severity_for_log_record(self, payload: dict[str, Any]) -> str: - status_code = self.runtime._int_or_none(payload.get("status_code")) - error = payload.get("error") - handled = error.get("handled") if isinstance(error, dict) else None - if status_code is not None and status_code >= 500: - return "ERROR" - if error is not None and handled is False: - return "ERROR" - if error is not None or ( - status_code is not None and status_code >= 400 - ): - return "WARNING" - return "INFO" - - def _int_or_none(self, value: Any) -> int | None: - try: - return int(value) - except (TypeError, ValueError): - return None - - def _internal_error_payload( - self, - operation: str, - exc: BaseException, - **fields: Any, - ) -> dict[str, Any]: - payload = { - "schema_version": "policyengine.observability.internal_error.v1", - "event": "observability_internal_error", - "service_name": self.runtime.config.service_name, - "service_role": self.runtime.config.service_role, - "environment": self.runtime.config.environment, - "created_at": datetime.now(UTC).isoformat(), - "operation": operation, - "error": { - "type": type(exc).__name__, - "message": self.runtime._safe_str(exc), - "stack": self.runtime._safe_traceback(exc), - }, - } - payload.update( - {key: value for key, value in fields.items() if value is not None} - ) - return payload - - def _safe_str(self, value: Any) -> str: - try: - return str(value) - except BaseException: - return f"" - - def _safe_traceback(self, exc: BaseException) -> str: - try: - return "".join( - traceback.format_exception(type(exc), exc, exc.__traceback__) - ) - except BaseException: - return "" - - def _json(self, payload: dict[str, Any]) -> str: - try: - return json.dumps(payload, sort_keys=True, default=str) - except BaseException: - return json.dumps( - { - "schema_version": "policyengine.observability.internal_error.v1", - "event": "observability_internal_error", - "created_at": datetime.now(UTC).isoformat(), - "operation": "observability.failure_json", - }, - sort_keys=True, - ) - - def _write_stderr(self, payload: dict[str, Any]) -> None: - try: - sys.stderr.write(self.runtime._json(payload) + "\n") - except BaseException: - return diff --git a/policyengine_observability/otel.py b/policyengine_observability/otel.py new file mode 100644 index 0000000..4cd91b2 --- /dev/null +++ b/policyengine_observability/otel.py @@ -0,0 +1,627 @@ +from __future__ import annotations + +import threading +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any, cast + +from .config import ObservabilityConfig, OTLPExporterConfig +from .diagnostics import Diagnostics +from .schema import error_fields, metric_attributes + + +@dataclass(slots=True) +class SpanHandle: + manager: Any + span: Any + + +class OTelRuntime: + def __init__( + self, + config: ObservabilityConfig, + diagnostics: Diagnostics, + *, + queue_depth: Callable[[], int] | None = None, + ) -> None: + self.config = config + self.diagnostics = diagnostics + self._queue_depth_callback = queue_depth or (lambda: 0) + self.tracer: Any | None = None + self.meter: Any | None = None + self._tracer_provider: Any | None = None + self._meter_provider: Any | None = None + self._owns_tracer_provider = False + self._owns_meter_provider = False + self._request_count: Any | None = None + self._request_duration: Any | None = None + self._operation_count: Any | None = None + self._operation_duration: Any | None = None + self._error_count: Any | None = None + self._dropped_count: Any | None = None + self._exporter_failure_count: Any | None = None + self._queue_depth: Any | None = None + self._configure() + + def _configure(self) -> None: + if not self.config.otel.enabled or not self.config.identity_complete: + return + try: + from opentelemetry import metrics, trace + + if self.config.otel.provider_mode == "external": + self._tracer_provider = trace.get_tracer_provider() + self._meter_provider = metrics.get_meter_provider() + else: + self._configure_owned_providers() + if self._tracer_provider is not None: + self.tracer = self._tracer_provider.get_tracer( + "policyengine-observability", "2.0.0" + ) + if self._meter_provider is not None: + self.meter = self._meter_provider.get_meter( + "policyengine-observability", "2.0.0" + ) + self._configure_instruments() + except Exception as exc: + self.diagnostics.report("otel.configure", exc) + self.tracer = None + self.meter = None + + def _configure_owned_providers(self) -> None: + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.sampling import ( + ParentBased, + TraceIdRatioBased, + ) + + resource = Resource.create(self.resource_attributes()) + readers: list[Any] = [] + otel = self.config.otel + metric_exporter = otel.metrics + if metric_exporter is not None: + from opentelemetry.sdk.metrics.export import ( + PeriodicExportingMetricReader, + ) + + readers.append( + PeriodicExportingMetricReader( + _lazy_metric_exporter_class()( + lambda: _build_metric_exporter(metric_exporter), + self.diagnostics, + ), + export_interval_millis=max( + 1_000, + otel.metric_export_interval_seconds * 1_000, + ), + export_timeout_millis=max( + 1, + metric_exporter.timeout_seconds * 1_000, + ), + ) + ) + self._meter_provider = MeterProvider( + metric_readers=readers, + resource=resource, + shutdown_on_exit=False, + ) + self._owns_meter_provider = True + + self._tracer_provider = TracerProvider( + sampler=ParentBased( + TraceIdRatioBased(min(max(otel.sampling_ratio, 0.0), 1.0)) + ), + resource=resource, + shutdown_on_exit=False, + meter_provider=self._meter_provider, + ) + self._owns_tracer_provider = True + trace_exporter = otel.traces + if trace_exporter is not None: + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + batch_size = min( + max(1, otel.span_batch_size), + max(1, otel.span_queue_capacity), + ) + self._tracer_provider.add_span_processor( + BatchSpanProcessor( + cast( + Any, + _LazySpanExporter( + lambda: _build_span_exporter(trace_exporter), + self.diagnostics, + ), + ), + max_queue_size=max(1, otel.span_queue_capacity), + max_export_batch_size=batch_size, + schedule_delay_millis=max( + 1, + otel.span_schedule_delay_seconds * 1_000, + ), + export_timeout_millis=max( + 1, + trace_exporter.timeout_seconds * 1_000, + ), + meter_provider=self._meter_provider, + ) + ) + + def _configure_instruments(self) -> None: + meter = self.meter + if meter is None: + return + self._request_count = meter.create_counter( + "policyengine.request.count" + ) + self._request_duration = meter.create_histogram( + "policyengine.request.duration", + unit="s", + ) + self._operation_count = meter.create_counter( + "policyengine.operation.count" + ) + self._operation_duration = meter.create_histogram( + "policyengine.operation.duration", + unit="s", + ) + self._error_count = meter.create_counter("policyengine.error.count") + self._dropped_count = meter.create_counter( + "policyengine.telemetry.dropped" + ) + self._exporter_failure_count = meter.create_counter( + "policyengine.telemetry.exporter.failure" + ) + self._queue_depth = meter.create_observable_gauge( + "policyengine.telemetry.queue.depth", + callbacks=[self._observe_queue_depth], + ) + + def _observe_queue_depth(self, _options: Any) -> list[Any]: + try: + from opentelemetry.metrics import Observation + + return [Observation(max(0, int(self._queue_depth_callback())))] + except Exception as exc: + self.diagnostics.report("otel.queue_depth", exc) + return [] + + def resource_attributes(self) -> dict[str, str]: + service = self.config.service + deployment = self.config.deployment + values = { + "service.name": service.name, + "service.namespace": service.namespace, + "service.version": service.version, + "service.instance.id": deployment.instance_id, + "deployment.environment.name": deployment.environment, + "cloud.platform": deployment.platform, + "cloud.region": deployment.region, + "policyengine.service.role": service.role, + } + return {key: value for key, value in values.items() if value} + + def start_span( + self, + name: str, + *, + kind: Any = None, + attributes: Mapping[str, Any] | None = None, + parent_context: Any = None, + links: Sequence[Any] | None = None, + ) -> SpanHandle | None: + if self.tracer is None: + return None + try: + kwargs: dict[str, Any] = { + "attributes": dict(attributes or {}), + "record_exception": False, + "set_status_on_exception": False, + } + if kind is not None: + kwargs["kind"] = kind + if parent_context is not None: + kwargs["context"] = parent_context + if links: + kwargs["links"] = list(links) + manager = self.tracer.start_as_current_span(name, **kwargs) + return SpanHandle(manager=manager, span=manager.__enter__()) + except Exception as exc: + self.diagnostics.report("otel.span_start", exc, span=name) + return None + + def end_span( + self, + handle: SpanHandle | None, + error: BaseException | None = None, + *, + failed: bool = False, + ) -> None: + if handle is None: + return + try: + if error is not None or failed: + from opentelemetry.trace import Status, StatusCode + + if error is not None: + details = error_fields(error, self.config) + handle.span.add_event( + "exception", + attributes={ + "exception.type": details["error.type"], + "exception.message": details["error.message"], + "exception.stacktrace": details["error.stack"], + "exception.escaped": False, + }, + ) + handle.span.set_status(Status(StatusCode.ERROR)) + handle.manager.__exit__( + type(error) if error is not None else None, + error, + error.__traceback__ if error is not None else None, + ) + except Exception as exc: + self.diagnostics.report("otel.span_end", exc) + + def set_span_attributes(self, values: Mapping[str, Any]) -> None: + try: + from opentelemetry import trace + + span = trace.get_current_span() + if not span.is_recording(): + return + for key, value in values.items(): + span.set_attribute(key, value) + except Exception as exc: + self.diagnostics.report("otel.span_attribute", exc) + + def current_correlation(self) -> dict[str, Any]: + try: + from opentelemetry import trace + + span_context = trace.get_current_span().get_span_context() + if not span_context.is_valid: + return {} + return { + "trace_id": format(span_context.trace_id, "032x"), + "span_id": format(span_context.span_id, "016x"), + "trace_sampled": bool(span_context.trace_flags.sampled), + } + except Exception: + return {} + + def extract(self, carrier: Mapping[str, str]) -> Any: + try: + from opentelemetry.trace.propagation.tracecontext import ( + TraceContextTextMapPropagator, + ) + + return TraceContextTextMapPropagator().extract(carrier=carrier) + except Exception as exc: + self.diagnostics.report("otel.context_extract", exc) + return None + + def inject(self, carrier: dict[str, str]) -> None: + try: + from opentelemetry.trace.propagation.tracecontext import ( + TraceContextTextMapPropagator, + ) + + TraceContextTextMapPropagator().inject(carrier=carrier) + except Exception as exc: + self.diagnostics.report("otel.context_inject", exc) + + def remote_span_context(self, carrier: Mapping[str, str]) -> Any | None: + try: + from opentelemetry import trace + + context = self.extract(carrier) + if context is None: + return None + span_context = trace.get_current_span(context).get_span_context() + return span_context if span_context.is_valid else None + except Exception: + return None + + def link(self, span_context: Any) -> Any | None: + if span_context is None: + return None + try: + from opentelemetry.trace import Link + + return Link(span_context) + except Exception: + return None + + def empty_context(self) -> Any | None: + try: + from opentelemetry.context import Context + + return Context() + except Exception: + return None + + def record_request( + self, + duration_seconds: float, + attributes: Mapping[str, Any], + ) -> None: + bounded = metric_attributes(attributes, self.config) + self._safe_metric(self._request_count, "add", 1, bounded) + self._safe_metric( + self._request_duration, + "record", + duration_seconds, + bounded, + ) + + def record_operation( + self, + duration_seconds: float, + attributes: Mapping[str, Any], + ) -> None: + bounded = metric_attributes(attributes, self.config) + self._safe_metric(self._operation_count, "add", 1, bounded) + self._safe_metric( + self._operation_duration, + "record", + duration_seconds, + bounded, + ) + + def record_error(self, attributes: Mapping[str, Any]) -> None: + self._safe_metric( + self._error_count, + "add", + 1, + metric_attributes(attributes, self.config), + ) + + def record_dropped(self, kind: str, count: int = 1) -> None: + self._safe_metric( + self._dropped_count, + "add", + count, + {"operation.kind": kind}, + ) + + def record_exporter_failure(self, kind: str, count: int = 1) -> None: + self._safe_metric( + self._exporter_failure_count, + "add", + count, + {"operation.kind": kind}, + ) + + def _safe_metric( + self, + instrument: Any | None, + method: str, + value: int | float, + attributes: Mapping[str, Any], + ) -> None: + if instrument is None: + return + try: + getattr(instrument, method)(value, attributes=dict(attributes)) + except Exception as exc: + self.diagnostics.report("otel.metric_record", exc) + + def force_flush(self, timeout_seconds: float) -> None: + timeout_ms = max(0, int(timeout_seconds * 1_000)) + for provider in (self._tracer_provider, self._meter_provider): + force_flush = getattr(provider, "force_flush", None) + if callable(force_flush): + try: + force_flush(timeout_millis=timeout_ms) + except Exception as exc: + self.diagnostics.report("otel.force_flush", exc) + + def shutdown(self, timeout_seconds: float) -> None: + timeout_ms = max(0, int(timeout_seconds * 1_000)) + if self._owns_tracer_provider and self._tracer_provider is not None: + try: + self._tracer_provider.shutdown() + except Exception as exc: + self.diagnostics.report("otel.trace_shutdown", exc) + if self._owns_meter_provider and self._meter_provider is not None: + try: + self._meter_provider.shutdown(timeout_millis=timeout_ms) + except Exception as exc: + self.diagnostics.report("otel.metric_shutdown", exc) + + +class _LazySpanExporter: + def __init__( + self, + factory: Callable[[], Any], + diagnostics: Diagnostics, + ) -> None: + self._factory = factory + self._diagnostics = diagnostics + self._delegate: Any | None = None + self._lock = threading.Lock() + + def export(self, spans: Sequence[Any]) -> Any: + from opentelemetry.sdk.trace.export import SpanExportResult + + try: + return self._get().export(spans) + except Exception as exc: + self._diagnostics.increment("spans.export_failure") + self._diagnostics.report("otel.span_export", exc) + return SpanExportResult.FAILURE + + def force_flush(self, timeout_millis: int = 30_000) -> bool: + try: + delegate = self._delegate + return ( + True + if delegate is None + else bool(delegate.force_flush(timeout_millis)) + ) + except Exception as exc: + self._diagnostics.report("otel.span_flush", exc) + return False + + def shutdown(self) -> None: + try: + if self._delegate is not None: + self._delegate.shutdown() + except Exception as exc: + self._diagnostics.report("otel.span_exporter_shutdown", exc) + + def _get(self) -> Any: + if self._delegate is None: + with self._lock: + if self._delegate is None: + self._delegate = self._factory() + return self._delegate + + +def _lazy_metric_exporter_class(): + from opentelemetry.sdk.metrics.export import MetricExporter + + class LazyMetricExporter(MetricExporter): + def __init__( + self, + factory: Callable[[], Any], + diagnostics: Diagnostics, + ) -> None: + super().__init__() + self._factory = factory + self._diagnostics = diagnostics + self._delegate: Any | None = None + self._lock = threading.Lock() + + def export( + self, + metrics_data: Any, + timeout_millis: float = 10_000, + **kwargs: Any, + ) -> Any: + from opentelemetry.sdk.metrics.export import MetricExportResult + + try: + return self._get().export( + metrics_data, + timeout_millis=timeout_millis, + **kwargs, + ) + except Exception as exc: + self._diagnostics.increment("metrics.export_failure") + self._diagnostics.report("otel.metric_export", exc) + return MetricExportResult.FAILURE + + def force_flush(self, timeout_millis: float = 10_000) -> bool: + try: + delegate = self._delegate + return ( + True + if delegate is None + else bool(delegate.force_flush(timeout_millis)) + ) + except Exception as exc: + self._diagnostics.report("otel.metric_flush", exc) + return False + + def shutdown( + self, timeout_millis: float = 30_000, **kwargs: Any + ) -> None: + try: + if self._delegate is not None: + self._delegate.shutdown( + timeout_millis=timeout_millis, + **kwargs, + ) + except Exception as exc: + self._diagnostics.report("otel.metric_exporter_shutdown", exc) + + def _get(self) -> Any: + if self._delegate is None: + with self._lock: + if self._delegate is None: + self._delegate = self._factory() + return self._delegate + + return LazyMetricExporter + + +def _build_span_exporter(config: OTLPExporterConfig) -> Any: + kwargs = _exporter_kwargs(config) + if config.protocol == "http/protobuf": + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, + ) + + if config.endpoint_mode == "base": + kwargs["endpoint"] = _http_signal_endpoint( + config.endpoint, "traces" + ) + return OTLPSpanExporter(**kwargs) + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, + ) + + return OTLPSpanExporter(**kwargs) + + +def _build_metric_exporter(config: OTLPExporterConfig) -> Any: + kwargs = _exporter_kwargs(config) + if config.protocol == "http/protobuf": + from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, + ) + + if config.endpoint_mode == "base": + kwargs["endpoint"] = _http_signal_endpoint( + config.endpoint, "metrics" + ) + return OTLPMetricExporter(**kwargs) + from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, + ) + + return OTLPMetricExporter(**kwargs) + + +def _exporter_kwargs(config: OTLPExporterConfig) -> dict[str, Any]: + headers = dict(config.headers) + kwargs: dict[str, Any] = { + "endpoint": config.endpoint, + "headers": headers, + "timeout": max(0.1, min(config.timeout_seconds, 60.0)), + } + if config.auth is not None: + kwargs.update( + config.auth.exporter_kwargs( + protocol=config.protocol, + headers=headers, + ) + ) + if "session" in kwargs: + kwargs.pop("headers", None) + return kwargs + + +def _http_signal_endpoint(endpoint: str, signal: str) -> str: + value = endpoint.rstrip("/") + if value.endswith(f"/v1/{signal}"): + return value + return f"{value}/v1/{signal}" + + +def captured_at_is_recent(value: Any, max_age_seconds: float) -> bool: + if not isinstance(value, str): + return False + try: + captured_at = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return False + if captured_at.tzinfo is None: + return False + age = (datetime.now(UTC) - captured_at.astimezone(UTC)).total_seconds() + return 0 <= age <= max_age_seconds diff --git a/policyengine_observability/runtime.py b/policyengine_observability/runtime.py index a2f7248..1b1d55c 100644 --- a/policyengine_observability/runtime.py +++ b/policyengine_observability/runtime.py @@ -1,540 +1,919 @@ -"""Public runtime interface and component lifecycle.""" - from __future__ import annotations +import inspect +import logging +import math +import re import threading import time -from collections.abc import AsyncIterator, Iterator -from enum import Enum +import uuid +from collections.abc import Mapping +from contextvars import ContextVar, Token +from dataclasses import dataclass, field +from datetime import UTC, datetime +from functools import wraps from typing import Any -from . import _state -from ._metrics import MetricRecorder, _NoOpInstrument -from ._operations import OperationLifecycle -from ._requests import RequestLifecycle -from ._state import ( - OBSERVABILITY_INTERNAL_DISPATCH_HEADER as OBSERVABILITY_INTERNAL_DISPATCH_HEADER, -) -from ._state import ( - REQUEST_ID_HEADER as REQUEST_ID_HEADER, -) -from ._state import ( - TRACEPARENT_HEADER as TRACEPARENT_HEADER, -) -from ._state import ( - ContextState, -) -from ._tracing import TraceRecorder from .config import ObservabilityConfig -from .context import OperationObservabilityContext, RequestObservabilityContext -from .destinations import LogDestinationManager -from .destinations.base import clamped -from .logging import ( - EVENT_LOGGER, - INTERNAL_LOGGER, - OPERATION_LOGGER, - REQUEST_LOGGER, - LogEmitter, -) -from .segments import SegmentRecorder +from .delivery import DeliveryManager +from .destinations import StdoutLogDestination +from .diagnostics import Diagnostics +from .otel import OTelRuntime, SpanHandle, captured_at_is_recent +from .schema import build_record, normalize_attributes + +REQUEST_ID_HEADER = "X-PolicyEngine-Request-Id" +TRACEPARENT_HEADER = "traceparent" +TRACESTATE_HEADER = "tracestate" + +_REQUEST_ID_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +@dataclass(slots=True) +class _RequestState: + request_id: str + method: str + route: str + start_time: float + span: SpanHandle | None + token: Token[Any] | None = None + status_code: int | None = None + attributes: dict[str, Any] = field(default_factory=dict) + error: BaseException | None = None + completed: bool = False + + +@dataclass(slots=True) +class _OperationState: + name: str + kind: str + request_id: str | None + start_time: float + span: SpanHandle | None + token: Token[Any] | None = None + attributes: dict[str, Any] = field(default_factory=dict) + error: BaseException | None = None + completed: bool = False + + +@dataclass(slots=True) +class _ChildSpanState: + name: str + start_time: float + span: SpanHandle | None class ObservabilityRuntime: - def __init__( - self, - config: ObservabilityConfig, - *, - segment_registry: type[Enum] | None = None, - ) -> None: + def __init__(self, config: ObservabilityConfig) -> None: + config.validate() self.config = config - self.segment_registry = segment_registry - self.enabled = config.enabled - self.trace = None - self.propagate = None - self.SpanKind = None - self.Status = None - self.StatusCode = None - self.tracer_provider = None - self.meter_provider = None - self.tracer = None - self.meter = None - self.operation_duration = _NoOpInstrument() - self.http_duration = _NoOpInstrument() - self.segment_duration = _NoOpInstrument() - self.calculate_duration = _NoOpInstrument() - self.backend_duration = _NoOpInstrument() - self.operations = _NoOpInstrument() - self.requests = _NoOpInstrument() - self.errors = _NoOpInstrument() - self.rate_limited = _NoOpInstrument() - self.failover_events = _NoOpInstrument() - self.active_requests = _NoOpInstrument() - self._httpx_instrumented = False - self._emitting_internal_error = False - self.log_destination_manager = LogDestinationManager( - config=config, - loggers={ - "request": REQUEST_LOGGER, - "operation": OPERATION_LOGGER, - "event": EVENT_LOGGER, - "internal": INTERNAL_LOGGER, - }, - serializer=self._json, - on_failure=self._handle_destination_failure, + self.diagnostics = Diagnostics( + sensitive_values=config.sensitive_values, ) - self._context_state = ContextState(self) - self._operations = OperationLifecycle(self) - self._requests = RequestLifecycle(self) - self._segments = SegmentRecorder(self) - self._logging = LogEmitter(self) - self._metrics = MetricRecorder(self) - self._tracing = TraceRecorder(self) - - @classmethod - def disabled(cls) -> ObservabilityRuntime: - return cls(ObservabilityConfig(enabled=False)) - - def configure(self) -> None: - self._configure_loggers() - if not self.enabled: - return - self.log_destination_manager.configure() - if not self.config.otel_enabled: - return - self._configure_otel() - if self.config.instrument_httpx: - self.instrument_httpx() - - def current_context(self) -> RequestObservabilityContext | None: - return self._context_state.current_context() + self._request_state: ContextVar[_RequestState | None] = ContextVar( + f"policyengine_request_{id(self)}", default=None + ) + self._operation_state: ContextVar[_OperationState | None] = ContextVar( + f"policyengine_operation_{id(self)}", default=None + ) + self._shutdown_lock = threading.Lock() + self._closed = False + self._delivery = self._new_delivery() + self._otel = self._new_otel() + self.diagnostics.add_listener(self._record_diagnostic_metric) + self._logging_handlers: list[ + tuple[logging.Logger, ObservabilityLogHandler] + ] = [] + for warning in config.diagnostics(): + self.diagnostics.report("configuration", warning) + if config.logging.capture_standard_library: + try: + instrument_logging( + logging.getLogger(), + self, + replace=config.logging.replace_existing_handlers, + ) + except Exception as exc: + self.diagnostics.report("logging.handler_install", exc) + + def _new_delivery(self) -> DeliveryManager: + try: + return DeliveryManager(self.config, self.diagnostics) + except Exception as exc: + self.diagnostics.report("delivery.configure", exc) + fallback = ObservabilityConfig( + service=self.config.service, + deployment=self.config.deployment, + logging=self.config.logging.__class__( + destinations=(StdoutLogDestination(),), + ), + otel=self.config.otel, + limits=self.config.limits, + application_attribute_keys=( + self.config.application_attribute_keys + ), + dispatch_attribute_keys=self.config.dispatch_attribute_keys, + metric_attribute_keys=self.config.metric_attribute_keys, + sensitive_values=self.config.sensitive_values, + ) + return DeliveryManager(fallback, self.diagnostics) - def current_operation(self) -> OperationObservabilityContext | None: - return self._context_state.current_operation() + def _new_otel(self) -> OTelRuntime: + try: + return OTelRuntime( + self.config, + self.diagnostics, + queue_depth=lambda: self._delivery.queue_depth, + ) + except Exception as exc: + self.diagnostics.report("otel.runtime", exc) + return OTelRuntime( + ObservabilityConfig( + service=self.config.service, + deployment=self.config.deployment, + logging=self.config.logging, + otel=self.config.otel.__class__(enabled=False), + limits=self.config.limits, + application_attribute_keys=( + self.config.application_attribute_keys + ), + dispatch_attribute_keys=( + self.config.dispatch_attribute_keys + ), + metric_attribute_keys=self.config.metric_attribute_keys, + sensitive_values=self.config.sensitive_values, + ), + self.diagnostics, + queue_depth=lambda: self._delivery.queue_depth, + ) - def operation(self, name: str, *, flavor: str | None = None, **attrs: Any): - return self._operations.operation(name, flavor=flavor, **attrs) + def _record_diagnostic_metric(self, name: str, value: int) -> None: + if "dropped" in name or name.endswith("queue_full"): + self._otel.record_dropped(name, value) + elif "export_failure" in name: + self._otel.record_exporter_failure(name, value) - def entrypoint( + def operation( self, - name: str | None = None, + name: str, *, - flavor: str | None = None, - **attrs: Any, - ): - return self._operations.entrypoint(name, flavor=flavor, **attrs) + attributes: Mapping[str, Any] | None = None, + remote_context: Mapping[str, Any] | None = None, + independent_retry: bool = False, + aggregate: bool = False, + ) -> _ScopeManager: + return _ScopeManager( + self, + scope_type="operation", + name=name, + attributes=attributes, + remote_context=remote_context, + independent_retry=independent_retry, + aggregate=aggregate, + ) - def start_operation( + def span( self, name: str, *, - flavor: str | None = None, - parent_context: Any = None, - timings: dict[str, float] | None = None, - emit_log: bool = True, - record_metric: bool = True, - **attrs: Any, - ) -> dict[str, Any]: - return self._operations.start_operation( - name, - flavor=flavor, - parent_context=parent_context, - timings=timings, - emit_log=emit_log, - record_metric=record_metric, - **attrs, + attributes: Mapping[str, Any] | None = None, + ) -> _ScopeManager: + return _ScopeManager( + self, + scope_type="span", + name=name, + attributes=attributes, ) - def end_operation( - self, handle: dict[str, Any] | None, error: BaseException | None = None - ) -> None: - return self._operations.end_operation(handle, error) - - def complete_operation( - self, operation: OperationObservabilityContext - ) -> None: - return self._operations.complete_operation(operation) + def _scope_copy(self, manager: _ScopeManager) -> _ScopeManager: + return _ScopeManager( + self, + scope_type=manager.scope_type, + name=manager.name, + attributes=manager.attributes, + remote_context=manager.remote_context, + independent_retry=manager.independent_retry, + aggregate=manager.aggregate, + ) def begin_request( - self, context: RequestObservabilityContext, *, carrier: Any = None - ) -> None: - return self._requests.begin_request(context, carrier=carrier) - - def _begin_request_operation(self, *args: Any, **kwargs: Any) -> Any: - return self._requests._begin_request_operation(*args, **kwargs) - - def finish_request(self, status_code: int) -> dict[str, str]: - return self._requests.finish_request(status_code) - - def prepare_response(self, status_code: int) -> dict[str, str]: - return self._requests.prepare_response(status_code) - - def complete_request(self, status_code: int | None = None) -> None: - return self._requests.complete_request(status_code) + self, + *, + headers: Mapping[str, str], + method: str, + route: str, + ) -> str: + request_id = _request_id(_header_value(headers, REQUEST_ID_HEADER)) + try: + parent = self._otel.extract(headers) + span = self._otel.start_span( + f"{method.upper()} {route}", + kind=_span_kind("SERVER"), + parent_context=parent, + attributes={ + "http.request.method": method.upper(), + "http.route": route, + }, + ) + state = _RequestState( + request_id=request_id, + method=method.upper(), + route=route, + start_time=time.perf_counter(), + span=span, + ) + state.token = self._request_state.set(state) + except Exception as exc: + self.diagnostics.report("request.begin", exc) + return request_id + + def update_request_route(self, route: str) -> None: + state = self._request_state.get() + if state is None or state.completed: + return + state.route = route + self._otel.set_span_attributes({"http.route": route}) - def update_request_route( - self, *, route: str | None = None, endpoint: str | None = None + def update_request_status(self, status_code: int) -> None: + state = self._request_state.get() + if state is None or state.completed: + return + state.status_code = status_code + + def response_headers(self) -> dict[str, str]: + state = self._request_state.get() + if state is None: + return {} + headers = {REQUEST_ID_HEADER: state.request_id} + carrier: dict[str, str] = {} + self._otel.inject(carrier) + if TRACEPARENT_HEADER in carrier: + headers[TRACEPARENT_HEADER] = carrier[TRACEPARENT_HEADER] + return headers + + def end_request( + self, + *, + status_code: int | None, + error: BaseException | None = None, ) -> None: - return self._requests.update_request_route( - route=route, endpoint=endpoint + state = self._request_state.get() + if state is None or state.completed: + return + state.completed = True + resolved_status = ( + status_code if status_code is not None else state.status_code ) + state.status_code = resolved_status + state.error = error + duration = max(0.0, time.perf_counter() - state.start_time) + outcome = _outcome(resolved_status, error) + metric_values = self._metric_base() + metric_values.update( + { + "http.route": state.route, + "http.request.method": state.method, + "http.response.status_code_class": _status_class( + resolved_status + ), + "operation.kind": "request", + "outcome": outcome, + } + ) + self._otel.set_span_attributes( + { + "http.route": state.route, + "http.response.status_code": resolved_status or 0, + "policyengine.request.id": state.request_id, + "policyengine.outcome": outcome, + } + ) + context = { + "request.id": state.request_id, + "http.request.method": state.method, + "http.route": state.route, + "http.response.status_code": resolved_status, + "duration_ms": round(duration * 1_000, 3), + "outcome": outcome, + **self._otel.current_correlation(), + } + self._emit_record( + severity="ERROR" if outcome == "error" else "INFO", + event_name="request.completed", + context=context, + attributes=state.attributes, + error=error, + ) + self._otel.record_request(duration, metric_values) + if outcome == "error": + self._otel.record_error(metric_values) + self._otel.end_span( + state.span, + error, + failed=outcome == "error", + ) + self._reset_request(state) - def teardown_request(self, exc: BaseException | None = None) -> None: - return self._requests.teardown_request(exc) - - def set_attribute(self, key: str, value: Any) -> None: - return self._requests.set_attribute(key, value) - - def segment(self, name: Any, **attrs: Any) -> Iterator[Any]: - return self._segments.segment(name, **attrs) - - def _segment_context(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._segment_context(*args, **kwargs) - - def asegment(self, name: Any, **attrs: Any) -> AsyncIterator[Any]: - return self._segments.asegment(name, **attrs) - - def collect_timings(self, name: str = "operation", **attrs: Any): - return self._operations.collect_timings(name, **attrs) - - def start_scope( + def set_context(self, **attributes: Any) -> None: + safe, omitted = normalize_attributes( + attributes, + self.config, + allowed_keys=( + self.config.application_attribute_keys + | self.config.dispatch_attribute_keys + ), + ) + request = self._request_state.get() + operation = self._operation_state.get() + target = operation or request + if target is not None: + target.attributes.update(safe) + if safe: + self._otel.set_span_attributes(safe) + if omitted: + self.diagnostics.increment("attributes.omitted", omitted) + + def event( self, - timings: dict[str, float], + name: str, *, - name: str = "operation", - parent_context: Any = None, - **attrs: Any, - ) -> dict[str, Any]: - return self._operations.start_scope( - timings, name=name, parent_context=parent_context, **attrs - ) - - def annotate( - self, handle: dict[str, Any] | None = None, **attrs: Any + severity: str = "INFO", + attributes: Mapping[str, Any] | None = None, ) -> None: - return self._operations.annotate(handle, **attrs) + self._emit_record( + severity=severity, + event_name=name, + context=self._active_context_fields(), + attributes=attributes, + ) - def end_scope( - self, handle: dict[str, Any] | None, error: BaseException | None = None + def log( + self, + message: str, + *, + severity: str = "INFO", + attributes: Mapping[str, Any] | None = None, + error: BaseException | None = None, ) -> None: - return self._operations.end_scope(handle, error) - - def mark(self, key: str, ms: float) -> None: - return self._operations.mark(key, ms) - - def mark_ttft(self, key: str = "ttft_ms") -> None: - return self._operations.mark_ttft(key) - - def mark_ttft_attribute(self, key: str = "ttft_ms") -> None: - return self._operations.mark_ttft_attribute(key) + self._emit_record( + severity=severity, + message=message, + context=self._active_context_fields(), + attributes=attributes, + error=error, + ) - def record_error( + def record_exception( self, - exc: BaseException, + error: Exception, *, handled: bool, status_code: int | None = None, - include_stack: bool = True, ) -> None: - return self._logging.record_error( - exc, - handled=handled, - status_code=status_code, - include_stack=include_stack, + request = self._request_state.get() + operation = self._operation_state.get() + if request is not None: + request.error = error + if status_code is not None: + request.status_code = status_code + if operation is not None: + operation.error = error + context = { + **self._active_context_fields(), + "error.handled": handled, + } + if status_code is not None: + context["http.response.status_code"] = status_code + self._emit_record( + severity="ERROR", + event_name="exception.recorded", + context=context, + error=error, + ) + self._otel.record_error( + { + **self._metric_base(), + "operation.kind": "handled" if handled else "unhandled", + "outcome": "error", + } ) - def record_event(self, event: str, **fields: Any) -> None: - return self._logging.record_event(event, **fields) - - def traceparent_header(self) -> str | None: - return self._tracing.traceparent_header() - - def capture_context(self): - return self._tracing.capture_context() - - def emit_request_log(self, context: RequestObservabilityContext) -> None: - return self._logging.emit_request_log(context) + def capture_context(self) -> dict[str, str]: + carrier: dict[str, str] = {} + self._otel.inject(carrier) + captured: dict[str, str] = { + key: value + for key, value in carrier.items() + if key.lower() in {TRACEPARENT_HEADER, TRACESTATE_HEADER} + } + captured["captured_at"] = ( + datetime.now(UTC).isoformat().replace("+00:00", "Z") + ) + request = self._request_state.get() + operation = self._operation_state.get() + if request is not None: + captured["request_id"] = request.request_id + elif operation is not None and operation.request_id: + captured["request_id"] = operation.request_id + source = operation.attributes if operation is not None else {} + if request is not None: + source = {**request.attributes, **source} + for key in self.config.dispatch_attribute_keys: + value = source.get(key) + if isinstance(value, (str, int)): + captured[key] = str(value)[ + : self.config.limits.max_string_length + ] + return captured + + def inject_http_headers(self, headers: dict[str, str]) -> None: + self._otel.inject(headers) + if REQUEST_ID_HEADER not in headers: + request = self._request_state.get() + operation = self._operation_state.get() + request_id = ( + request.request_id + if request is not None + else operation.request_id + if operation is not None + else None + ) + if request_id: + headers[REQUEST_ID_HEADER] = request_id - def emit_operation_log( - self, operation: OperationObservabilityContext - ) -> None: - return self._logging.emit_operation_log(operation) + def shutdown(self) -> None: + with self._shutdown_lock: + if self._closed: + return + self._closed = True + logging_timeout = _bounded_shutdown_timeout( + self.config.logging.shutdown_timeout_seconds, + default=2.0, + ) + otel_timeout = _bounded_shutdown_timeout( + self.config.otel.shutdown_timeout_seconds, + default=3.0, + ) + try: + try: + self._delivery.close(logging_timeout) + except Exception as exc: + self.diagnostics.report("logging.shutdown", exc) + try: + _run_bounded( + lambda: self._otel.shutdown(otel_timeout), + otel_timeout, + self.diagnostics, + "otel.shutdown_deadline", + ) + except Exception as exc: + self.diagnostics.report("otel.shutdown", exc) + finally: + self._remove_logging_handlers() + + def restart_after_snapshot(self) -> None: + """Rebuild process-local state after a fork or snapshot restore. + + Call this only in a single-threaded lifecycle callback before the + copied process accepts application work. Inherited workers and locks + are abandoned because their owning threads may not exist in the new + process. + """ - def record_operation_metric( - self, duration_seconds: float, attributes: dict[str, str] - ) -> None: - return self._metrics.record_operation_metric( - duration_seconds, attributes + self._shutdown_lock = threading.Lock() + self.diagnostics.restart_after_process_duplication() + self._request_state = ContextVar( + f"policyengine_request_{id(self)}", default=None ) - - def record_request_metric( - self, duration_seconds: float, attributes: dict[str, str] - ) -> None: - return self._metrics.record_request_metric( - duration_seconds, attributes + self._operation_state = ContextVar( + f"policyengine_operation_{id(self)}", default=None + ) + for _logger, handler in self._logging_handlers: + try: + handler.createLock() + except Exception as exc: + self.diagnostics.report("process.logging_handler_lock", exc) + self._delivery = self._new_delivery() + self._otel = self._new_otel() + self._closed = False + + def _start_operation( + self, + name: str, + attributes: Mapping[str, Any] | None, + remote_context: Mapping[str, Any] | None, + independent_retry: bool, + aggregate: bool, + ) -> _OperationState: + safe, omitted = normalize_attributes( + attributes, + self.config, + allowed_keys=( + self.config.application_attribute_keys + | self.config.dispatch_attribute_keys + ), ) + if omitted: + self.diagnostics.increment("attributes.omitted", omitted) + parent = None + links: list[Any] = [] + request_id: str | None = None + if remote_context: + request_id = _valid_request_id(remote_context.get("request_id")) + carrier = { + key: str(value) + for key, value in remote_context.items() + if key.lower() in {TRACEPARENT_HEADER, TRACESTATE_HEADER} + } + extracted = self._otel.extract(carrier) + direct = ( + not independent_retry + and not aggregate + and captured_at_is_recent( + remote_context.get("captured_at"), + self.config.limits.async_parent_max_age_seconds, + ) + ) + if direct: + parent = extracted + else: + link = self._otel.link(self._otel.remote_span_context(carrier)) + if link is not None: + links.append(link) + parent = self._otel.empty_context() + active_request = self._request_state.get() + if request_id is None and active_request is not None: + request_id = active_request.request_id + span = self._otel.start_span( + name, + kind=_span_kind("CONSUMER") if remote_context else None, + attributes={ + "operation.name": name, + "operation.kind": "operation", + **safe, + }, + parent_context=parent, + links=links, + ) + state = _OperationState( + name=name, + kind="operation", + request_id=request_id, + start_time=time.perf_counter(), + span=span, + attributes=safe, + ) + state.token = self._operation_state.set(state) + return state - def record_segment_metric( + def _finish_operation( self, - segment: str, - duration_seconds: float, - attributes: dict[str, str], - *, - backend_segment: bool = False, + state: _OperationState | None, + error: BaseException | None, ) -> None: - return self._metrics.record_segment_metric( - segment, - duration_seconds, + if state is None or state.completed: + return + state.completed = True + state.error = error or state.error + duration = max(0.0, time.perf_counter() - state.start_time) + outcome = "error" if state.error is not None else "success" + context = { + "operation.name": state.name, + "operation.kind": state.kind, + "request.id": state.request_id, + "duration_ms": round(duration * 1_000, 3), + "outcome": outcome, + **self._otel.current_correlation(), + } + self._emit_record( + severity="ERROR" if state.error is not None else "INFO", + event_name="operation.completed", + context=context, + attributes=state.attributes, + error=state.error, + ) + metrics = { + **self._metric_base(), + "operation.name": state.name, + "operation.kind": state.kind, + "outcome": outcome, + } + self._otel.record_operation(duration, metrics) + if state.error is not None: + self._otel.record_error(metrics) + self._otel.end_span(state.span, state.error) + if state.token is not None: + try: + self._operation_state.reset(state.token) + except Exception as exc: + self.diagnostics.report("operation.context_reset", exc) + + def _start_child_span( + self, + name: str, + attributes: Mapping[str, Any] | None, + ) -> _ChildSpanState: + safe, omitted = normalize_attributes( attributes, - backend_segment=backend_segment, + self.config, + allowed_keys=( + self.config.application_attribute_keys + | self.config.dispatch_attribute_keys + ), + ) + if omitted: + self.diagnostics.increment("attributes.omitted", omitted) + return _ChildSpanState( + name=name, + start_time=time.perf_counter(), + span=self._otel.start_span(name, attributes=safe), ) - def record_error_metric(self, attributes: dict[str, str]) -> None: - return self._metrics.record_error_metric(attributes) - - def record_rate_limited_metric(self, attributes: dict[str, str]) -> None: - return self._metrics.record_rate_limited_metric(attributes) - - def record_failover_event_metric(self, attributes: dict[str, str]) -> None: - return self._metrics.record_failover_event_metric(attributes) - - def record_active_request( - self, delta: int, attributes: dict[str, str] + def _finish_child_span( + self, + state: _ChildSpanState | None, + error: BaseException | None, ) -> None: - return self._metrics.record_active_request(delta, attributes) - - def instrument_fastapi(self, app: Any) -> None: - return self._tracing.instrument_fastapi(app) - - def instrument_httpx(self) -> None: - return self._tracing.instrument_httpx() - - def shutdown(self) -> None: - budget = clamped( - self.config.shutdown_timeout_seconds, - low=0.0, - high=60.0, - default=ObservabilityConfig.shutdown_timeout_seconds, - ) - providers = [ - ("trace", self.tracer_provider), - ("metrics", self.meter_provider), - ] - providers = [ - (name, provider) - for name, provider in providers - if provider is not None - ] - # Destination close is inherently deadline-bounded, so it runs - # inline and first, with a deadline that leaves room for the - # provider flush when there is one. Everything below fits inside - # the one shutdown budget by construction. - started = time.monotonic() - try: - self.log_destination_manager.close( - budget / 2 if providers else budget - ) - except BaseException as exc: - self.log_observability_failure("logging.destination_close", exc) - if not providers: + if state is None: return - remaining = max(0.0, budget - (time.monotonic() - started)) - - def flush() -> None: - for name, provider in providers: - try: - provider.shutdown() - except BaseException as exc: - self.log_observability_failure( - f"otel.{name}_shutdown", - exc, - ) - - thread = threading.Thread( - target=flush, - name="policyengine-otel-shutdown", - daemon=True, - ) - thread.start() - thread.join(timeout=remaining) - if thread.is_alive(): - self.log_observability_failure( - "otel.shutdown_timeout", - TimeoutError("OpenTelemetry shutdown timed out."), - timeout_seconds=remaining, + self._otel.end_span(state.span, error) + + def _active_context_fields(self) -> dict[str, Any]: + fields: dict[str, Any] = {} + request = self._request_state.get() + operation = self._operation_state.get() + if request is not None: + fields.update( + { + "request.id": request.request_id, + "http.request.method": request.method, + "http.route": request.route, + } ) + if operation is not None: + fields.update( + { + "operation.name": operation.name, + "operation.kind": operation.kind, + } + ) + if operation.request_id and "request.id" not in fields: + fields["request.id"] = operation.request_id + fields.update(self._otel.current_correlation()) + return fields + + def _metric_base(self) -> dict[str, Any]: + return { + "service.name": self.config.service.name, + "service.role": self.config.service.role, + "deployment.environment.name": ( + self.config.deployment.environment + ), + "cloud.platform": self.config.deployment.platform, + } + + def _emit_record(self, **kwargs: Any) -> None: + try: + self._delivery.emit(build_record(self.config, **kwargs)) + except Exception as exc: + self.diagnostics.report("record.emit", exc) - def shutdown_tracing(self) -> None: - self.shutdown() - - def restart_log_destinations(self) -> None: - """Close and rebuild log destinations from configuration. - - Call ONLY from single-threaded lifecycle moments — a - post-snapshot-restore hook, a post-fork hook, before serving - traffic. There is deliberately no locking here: under that - contract there is no concurrency, and a violated contract costs - at most a counted drop into a closing destination. - - A no-op when observability is disabled, mirroring configure(): - the kill switch must hold across forks and snapshot restores. - """ - if not self.enabled: + def _reset_request(self, state: _RequestState) -> None: + if state.token is None: return - self.log_destination_manager.configure() + try: + self._request_state.reset(state.token) + except Exception as exc: + self.diagnostics.report("request.context_reset", exc) - def log_observability_failure( - self, operation: str, exc: BaseException, **fields: Any + def _register_logging_handler( + self, + logger: logging.Logger, + handler: ObservabilityLogHandler, ) -> None: - return self._logging.log_observability_failure( - operation, exc, **fields - ) - - def _configure_loggers(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._configure_loggers(*args, **kwargs) - - def _emit_structured_log(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._emit_structured_log(*args, **kwargs) - - def _handle_destination_failure(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._handle_destination_failure(*args, **kwargs) - - def _severity_for_log_record(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._severity_for_log_record(*args, **kwargs) + self._logging_handlers.append((logger, handler)) - def _int_or_none(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._int_or_none(*args, **kwargs) + def _remove_logging_handlers(self) -> None: + for logger, handler in self._logging_handlers: + try: + logger.removeHandler(handler) + except Exception as exc: + self.diagnostics.report("logging.handler_remove", exc) + self._logging_handlers.clear() - def _internal_error_payload(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._internal_error_payload(*args, **kwargs) - def _configure_otel(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._configure_otel(*args, **kwargs) - - def _add_trace_exporter(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._add_trace_exporter(*args, **kwargs) - - def _metric_reader(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._metric_reader(*args, **kwargs) - - def _configure_instruments(self, *args: Any, **kwargs: Any) -> Any: - return self._metrics._configure_instruments(*args, **kwargs) - - def _instrument(self, *args: Any, **kwargs: Any) -> Any: - return self._metrics._instrument(*args, **kwargs) - - def _start_request_span(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._start_request_span(*args, **kwargs) - - def _close_request_span(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._close_request_span(*args, **kwargs) - - def _safe_span(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._safe_span(*args, **kwargs) - - def _start_span(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._start_span(*args, **kwargs) - - def _end_span(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._end_span(*args, **kwargs) - - def _start_segment_tree_node(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._start_segment_tree_node(*args, **kwargs) - - def _finish_segment_tree_node(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._finish_segment_tree_node(*args, **kwargs) - - def _reset_segment_tree_stack(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._reset_segment_tree_stack(*args, **kwargs) - - def _segment_tree_owner(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._segment_tree_owner(*args, **kwargs) - - def _safe_segment_tree_attrs(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._safe_segment_tree_attrs(*args, **kwargs) - - def _record_segment_flat_timing(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._record_segment_flat_timing(*args, **kwargs) - - def _record_segment_safely(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._record_segment_safely(*args, **kwargs) - - def _record_timing(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._record_timing(*args, **kwargs) - - def _segment_span_attributes(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._segment_span_attributes(*args, **kwargs) - - def _span_name(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._span_name(*args, **kwargs) - - def _start_implicit_operation(self, *args: Any, **kwargs: Any) -> Any: - return self._operations._start_implicit_operation(*args, **kwargs) - - def _coerce_segment(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._coerce_segment(*args, **kwargs) - - def _set_current_span_attributes(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._set_current_span_attributes(*args, **kwargs) - - def _current_span(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._current_span(*args, **kwargs) - - def _trace_ids(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._trace_ids(*args, **kwargs) - - def _extract_context(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._extract_context(*args, **kwargs) - - def _record_exception_on_span(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._record_exception_on_span(*args, **kwargs) - - def _add_span_event(self, *args: Any, **kwargs: Any) -> Any: - return self._tracing._add_span_event(*args, **kwargs) +class _ScopeManager: + def __init__( + self, + runtime: ObservabilityRuntime, + *, + scope_type: str, + name: str, + attributes: Mapping[str, Any] | None, + remote_context: Mapping[str, Any] | None = None, + independent_retry: bool = False, + aggregate: bool = False, + ) -> None: + self.runtime = runtime + self.scope_type = scope_type + self.name = str(name) + self.attributes = attributes + self.remote_context = remote_context + self.independent_retry = independent_retry + self.aggregate = aggregate + self.state: _OperationState | _ChildSpanState | None = None + + def __enter__(self) -> Any: + try: + if self.scope_type == "operation": + self.state = self.runtime._start_operation( + self.name, + self.attributes, + self.remote_context, + self.independent_retry, + self.aggregate, + ) + else: + self.state = self.runtime._start_child_span( + self.name, self.attributes + ) + except Exception as exc: + self.runtime.diagnostics.report( + f"{self.scope_type}.start", exc, name=self.name + ) + self.state = None + return self.state - def _close_active_request(self, *args: Any, **kwargs: Any) -> Any: - return self._requests._close_active_request(*args, **kwargs) + def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> bool: + try: + if self.scope_type == "operation": + self.runtime._finish_operation(self.state, exc) # type: ignore[arg-type] + else: + self.runtime._finish_child_span(self.state, exc) # type: ignore[arg-type] + except Exception as observability_error: + self.runtime.diagnostics.report( + f"{self.scope_type}.finish", + observability_error, + name=self.name, + ) + return False - def _reset_request_operation_context( - self, *args: Any, **kwargs: Any - ) -> Any: - return self._requests._reset_request_operation_context(*args, **kwargs) + async def __aenter__(self) -> Any: + return self.__enter__() - def _reset_request_context(self, *args: Any, **kwargs: Any) -> Any: - return self._requests._reset_request_context(*args, **kwargs) + async def __aexit__(self, exc_type: Any, exc: Any, traceback: Any) -> bool: + return self.__exit__(exc_type, exc, traceback) - def _safe_perf_counter(self, *args: Any, **kwargs: Any) -> Any: - return self._segments._safe_perf_counter(*args, **kwargs) + def __call__(self, function: Any) -> Any: + if inspect.iscoroutinefunction(function): - def _safe_str(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._safe_str(*args, **kwargs) + @wraps(function) + async def async_wrapper(*args: Any, **kwargs: Any) -> Any: + async with self.runtime._scope_copy(self): + return await function(*args, **kwargs) - def _safe_traceback(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._safe_traceback(*args, **kwargs) + return async_wrapper - def _json(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._json(*args, **kwargs) + @wraps(function) + def wrapper(*args: Any, **kwargs: Any) -> Any: + with self.runtime._scope_copy(self): + return function(*args, **kwargs) - def _write_stderr(self, *args: Any, **kwargs: Any) -> Any: - return self._logging._write_stderr(*args, **kwargs) + return wrapper -_RUNTIME = ObservabilityRuntime(ObservabilityConfig()) +class ObservabilityLogHandler(logging.Handler): + _IGNORED_PREFIXES = ( + "google.", + "grpc", + "opentelemetry.", + "policyengine_observability.", + ) + def __init__(self, runtime: ObservabilityRuntime) -> None: + super().__init__(level=runtime.config.logging.minimum_severity) + self.runtime = runtime -def set_observability_runtime(runtime: ObservabilityRuntime) -> None: - global _RUNTIME - _RUNTIME = runtime - for context_var in ( - _state._REQUEST_CONTEXT, - _state._OPERATION_CONTEXT, - _state._TIMINGS, - _state._TURN_START, - ): + def emit(self, record: logging.LogRecord) -> None: + if record.name.startswith(self._IGNORED_PREFIXES): + return try: - context_var.set(None) - except BaseException: - continue + attributes = getattr(record, "policyengine_attributes", None) + error = record.exc_info[1] if record.exc_info else None + self.runtime.log( + record.getMessage(), + severity=record.levelname, + attributes=attributes + if isinstance(attributes, Mapping) + else None, + error=error, + ) + except Exception as exc: + self.runtime.diagnostics.report("logging.handler_emit", exc) + + +def instrument_logging( + logger: logging.Logger, + runtime: ObservabilityRuntime, + *, + replace: bool = False, +) -> ObservabilityLogHandler: + for handler in logger.handlers: + if ( + isinstance(handler, ObservabilityLogHandler) + and handler.runtime is runtime + ): + return handler + handler = ObservabilityLogHandler(runtime) + previous_handlers = list(logger.handlers) + try: + if replace: + logger.handlers.clear() + logger.addHandler(handler) + except Exception: + if replace: + logger.handlers[:] = previous_handlers + raise + runtime._register_logging_handler(logger, handler) + return handler + + +def configure(config: ObservabilityConfig) -> ObservabilityRuntime: + return ObservabilityRuntime(config) + + +def _request_id(value: Any) -> str: + return _valid_request_id(value) or str(uuid.uuid4()) + + +def _header_value(headers: Mapping[str, str], name: str) -> str | None: + lowered = name.lower() + for key, value in headers.items(): + if key.lower() == lowered: + return value + return None + + +def _valid_request_id(value: Any) -> str | None: + if isinstance(value, str) and _REQUEST_ID_PATTERN.fullmatch(value): + return value + return None + + +def _status_class(status_code: int | None) -> str: + if status_code is None or status_code < 100 or status_code > 599: + return "unknown" + return f"{status_code // 100}xx" + + +def _outcome(status_code: int | None, error: BaseException | None) -> str: + if error is not None or (status_code is not None and status_code >= 500): + return "error" + if status_code is not None and status_code >= 400: + return "client_error" + return "success" + + +def _span_kind(name: str) -> Any: + try: + from opentelemetry.trace import SpanKind + + return getattr(SpanKind, name) + except Exception: + return None + + +def _run_bounded( + function: Any, + timeout_seconds: float, + diagnostics: Diagnostics, + diagnostic_name: str, +) -> None: + completed = threading.Event() + + def run() -> None: + try: + function() + except Exception as exc: + diagnostics.report(diagnostic_name, exc) + finally: + completed.set() + + thread = threading.Thread(target=run, daemon=True) + thread.start() + completed.wait(max(0.0, timeout_seconds)) + if not completed.is_set(): + diagnostics.increment("shutdown.timeout") + diagnostics.report( + diagnostic_name, + "Operation exceeded the configured shutdown deadline.", + ) -def observability_runtime() -> ObservabilityRuntime: - return _RUNTIME +def _bounded_shutdown_timeout(value: float, *, default: float) -> float: + try: + parsed = float(value) + except (TypeError, ValueError): + return default + if not math.isfinite(parsed): + return default + return min(max(parsed, 0.0), 60.0) diff --git a/policyengine_observability/schema.py b/policyengine_observability/schema.py new file mode 100644 index 0000000..7133b1c --- /dev/null +++ b/policyengine_observability/schema.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +import math +import traceback +from collections.abc import Mapping +from datetime import UTC, datetime +from enum import Enum +from typing import Any + +from .config import ObservabilityConfig + +SCHEMA_VERSION = "policyengine.observability.v2" + +_PROHIBITED_KEY_PARTS = ( + "authorization", + "cookie", + "credential", + "password", + "secret", + "token", + "request_body", + "response_body", + "household", + "person", + "reform", + "payload", + "raw_ip", + "client_ip", + "prompt", + "model_response", +) + + +def normalize_attributes( + values: Mapping[str, Any] | None, + config: ObservabilityConfig, + *, + allowed_keys: frozenset[str] | None = None, +) -> tuple[dict[str, str | int | float | bool], int]: + normalized: dict[str, str | int | float | bool] = {} + omitted = 0 + for raw_key, value in (values or {}).items(): + key = str(raw_key).strip() + if ( + not key + or len(normalized) >= config.limits.max_attributes + or _prohibited_key(key) + or (allowed_keys is not None and key not in allowed_keys) + ): + omitted += 1 + continue + scalar = _normalize_scalar(value, config) + if scalar is None: + omitted += 1 + continue + normalized[key] = scalar + return normalized, omitted + + +def build_record( + config: ObservabilityConfig, + *, + severity: str, + event_name: str | None = None, + message: str | None = None, + context: Mapping[str, Any] | None = None, + attributes: Mapping[str, Any] | None = None, + error: BaseException | None = None, +) -> dict[str, Any]: + record: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "timestamp": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + "severity": severity.upper(), + "service.name": _limit(config.service.name, config), + "service.namespace": _limit(config.service.namespace, config), + "service.version": _limit(config.service.version, config), + "service.role": _limit(config.service.role, config), + "deployment.environment.name": _limit( + config.deployment.environment, config + ), + "cloud.platform": config.deployment.platform, + } + if config.deployment.region: + record["cloud.region"] = _limit(config.deployment.region, config) + if config.deployment.instance_id: + record["service.instance.id"] = _limit( + config.deployment.instance_id, config + ) + if event_name: + record["event.name"] = _limit(event_name, config) + if message: + record["message"] = _redact(message, config)[ + : config.limits.max_string_length + ] + + for key, value in (context or {}).items(): + if value is not None: + record[str(key)] = value + + safe_attributes, omitted = normalize_attributes( + attributes, + config, + allowed_keys=config.application_attribute_keys + | config.dispatch_attribute_keys, + ) + if safe_attributes: + record["attributes"] = safe_attributes + if omitted: + record["attributes.omitted_count"] = omitted + + if error is not None: + record.update(error_fields(error, config)) + + return record + + +def metric_attributes( + values: Mapping[str, Any], config: ObservabilityConfig +) -> dict[str, str | int | float | bool]: + normalized, _ = normalize_attributes( + values, + config, + allowed_keys=config.metric_attribute_keys, + ) + return normalized + + +def error_fields( + error: BaseException, config: ObservabilityConfig +) -> dict[str, Any]: + try: + message = str(error) + except Exception: + message = "" + try: + stack = "".join( + traceback.format_exception(type(error), error, error.__traceback__) + ) + except Exception: + stack = "" + return { + "error.type": type(error).__name__, + "error.message": _redact(message, config)[ + : config.limits.max_error_message_length + ], + "error.stack": _redact(stack, config)[ + : config.limits.max_stack_length + ], + } + + +def _normalize_scalar( + value: Any, config: ObservabilityConfig +) -> str | int | float | bool | None: + if value is None: + return None + if isinstance(value, Enum): + value = value.value + if isinstance(value, bool): + return value + if isinstance(value, int): + return value + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, str): + return _redact(value, config)[: config.limits.max_string_length] + return None + + +def _prohibited_key(key: str) -> bool: + lowered = key.lower() + return any(part in lowered for part in _PROHIBITED_KEY_PARTS) + + +def _limit(value: Any, config: ObservabilityConfig) -> str: + try: + return str(value)[: config.limits.max_string_length] + except Exception: + return "" + + +def _redact(value: str, config: ObservabilityConfig) -> str: + redacted = value + for sensitive in config.sensitive_values: + if sensitive: + redacted = redacted.replace(sensitive, "[REDACTED]") + return redacted diff --git a/policyengine_observability/segments.py b/policyengine_observability/segments.py deleted file mode 100644 index 588ed9d..0000000 --- a/policyengine_observability/segments.py +++ /dev/null @@ -1,432 +0,0 @@ -"""Segment names, nested timings, and context managers.""" - -from __future__ import annotations - -import inspect -import time -from collections.abc import AsyncIterator, Iterable, Iterator -from contextlib import asynccontextmanager, contextmanager -from enum import Enum -from functools import wraps -from typing import TYPE_CHECKING, Any - -from . import _state -from .context import ( - OperationObservabilityContext, - RequestObservabilityContext, - SegmentTimingNode, - _metric_attrs, -) - -if TYPE_CHECKING: - from .runtime import ObservabilityRuntime - -UNKNOWN_SEGMENT = "unknown_segment" - - -def segment_values( - registry: type[Enum] | Iterable[str] | None, -) -> frozenset[str]: - if registry is None: - return frozenset() - if isinstance(registry, type) and issubclass(registry, Enum): - return frozenset(str(member.value) for member in registry) - return frozenset(str(value) for value in registry) - - -def coerce_segment_name( - value: Any, - *, - registry: type[Enum] | Iterable[str] | None = None, -) -> tuple[str, bool]: - values = segment_values(registry) - if isinstance(value, Enum): - segment = str(value.value) - return segment, not values or segment in values - if isinstance(value, str): - return value, not values or value in values - try: - segment = str(value) - except BaseException: - return UNKNOWN_SEGMENT, False - return segment, False if values else True - - -MAX_SEGMENT_ATTR_LENGTH = 200 -SENSITIVE_SEGMENT_ATTR_PARTS = ( - "authorization", - "credential", - "password", - "secret", - "token", -) - - -class SegmentRecorder: - def __init__(self, runtime: ObservabilityRuntime) -> None: - self.runtime = runtime - - def segment(self, name: Any, **attrs: Any) -> Iterator[Any]: - return _SegmentManager(self.runtime, name, attrs) - - @contextmanager - def _segment_context(self, name: Any, **attrs: Any) -> Iterator[Any]: - if not self.runtime.enabled: - yield None - return - segment_name = self.runtime._coerce_segment(name) - implicit_operation = self.runtime._start_implicit_operation( - segment_name, - attrs, - ) - start = self.runtime._safe_perf_counter( - f"segment.{segment_name}.start" - ) - segment_tree_handle = self.runtime._start_segment_tree_node( - segment_name, - attrs, - ) - span_attrs = self.runtime._segment_span_attributes(attrs) - span_name = self.runtime._span_name(segment_name) - error: BaseException | None = None - with self.runtime._safe_span(span_name, span_attrs) as span: - try: - yield span - except BaseException as exc: - error = exc - self.runtime._record_segment_safely( - segment_name, - start, - attrs, - segment_tree_handle=segment_tree_handle, - ) - raise - else: - self.runtime._record_segment_safely( - segment_name, - start, - attrs, - segment_tree_handle=segment_tree_handle, - ) - finally: - self.runtime._reset_segment_tree_stack(segment_tree_handle) - self.runtime.end_operation(implicit_operation, error) - - @asynccontextmanager - async def asegment(self, name: Any, **attrs: Any) -> AsyncIterator[Any]: - if not self.runtime.enabled: - yield None - return - segment_name = self.runtime._coerce_segment(name) - implicit_operation = self.runtime._start_implicit_operation( - segment_name, - attrs, - ) - start = self.runtime._safe_perf_counter( - f"segment.{segment_name}.start" - ) - segment_tree_handle = self.runtime._start_segment_tree_node( - segment_name, - attrs, - ) - span_attrs = self.runtime._segment_span_attributes(attrs) - span_name = self.runtime._span_name(segment_name) - error: BaseException | None = None - with self.runtime._safe_span(span_name, span_attrs) as span: - try: - yield span - except BaseException as exc: - error = exc - self.runtime._record_segment_safely( - segment_name, - start, - attrs, - segment_tree_handle=segment_tree_handle, - ) - raise - else: - self.runtime._record_segment_safely( - segment_name, - start, - attrs, - segment_tree_handle=segment_tree_handle, - ) - finally: - self.runtime._reset_segment_tree_stack(segment_tree_handle) - self.runtime.end_operation(implicit_operation, error) - - def _start_segment_tree_node( - self, - name: str, - attrs: dict[str, Any], - ) -> dict[str, Any] | None: - try: - owner = self.runtime._segment_tree_owner() - if owner is None: - return None - owner.segment_sequence[0] += 1 - node = SegmentTimingNode( - sequence=owner.segment_sequence[0], - name=name, - attrs=self.runtime._safe_segment_tree_attrs(attrs), - ) - owner_id = id(owner.segment_tree) - stack = _state._SEGMENT_STACK.get() - if stack and stack[-1][0] == owner_id: - stack[-1][1].children.append(node) - else: - owner.segment_tree.append(node) - token = _state._SEGMENT_STACK.set((*stack, (owner_id, node))) - return {"node": node, "token": token} - except BaseException as exc: - self.runtime.log_observability_failure( - "segment.tree_start", - exc, - segment=name, - ) - return None - - def _finish_segment_tree_node( - self, - handle: dict[str, Any] | None, - duration_seconds: float, - ) -> None: - if not handle: - return - try: - node = handle.get("node") - if not isinstance(node, SegmentTimingNode): - return - node.duration_ms = duration_seconds * 1000 - except BaseException as exc: - self.runtime.log_observability_failure( - "segment.tree_finish", - exc, - ) - - def _reset_segment_tree_stack( - self, - handle: dict[str, Any] | None, - ) -> None: - if not handle: - return - token = handle.get("token") - if token is None: - return - try: - _state._SEGMENT_STACK.reset(token) - except BaseException as exc: - self.runtime.log_observability_failure("segment.tree_reset", exc) - - def _segment_tree_owner( - self, - ) -> RequestObservabilityContext | OperationObservabilityContext | None: - context = self.runtime.current_context() - if context is not None: - return context - return self.runtime.current_operation() - - def _safe_segment_tree_attrs( - self, - attrs: dict[str, Any], - ) -> dict[str, Any]: - safe_attrs: dict[str, Any] = {} - for key, value in attrs.items(): - key_text = self.runtime._safe_str(key) - key_lower = key_text.lower() - if any(part in key_lower for part in SENSITIVE_SEGMENT_ATTR_PARTS): - continue - if value is None: - continue - if hasattr(value, "value"): - value = value.value - if isinstance(value, bool | int | float): - safe_attrs[key_text] = value - elif isinstance(value, str): - safe_attrs[key_text] = value[:MAX_SEGMENT_ATTR_LENGTH] - return safe_attrs - - def _record_segment_flat_timing( - self, - context: RequestObservabilityContext | None, - operation: OperationObservabilityContext | None, - name: str, - duration_ms: float, - ) -> None: - seen_timing_ids: set[int] = set() - seen_count_ids: set[int] = set() - for target in (context, operation): - if target is None: - continue - timings_id = id(target.timings_ms) - if timings_id not in seen_timing_ids: - target.timings_ms[name] = round( - target.timings_ms.get(name, 0.0) + duration_ms, - 3, - ) - seen_timing_ids.add(timings_id) - counts_id = id(target.timing_counts) - if counts_id not in seen_count_ids: - target.timing_counts[name] = ( - target.timing_counts.get(name, 0) + 1 - ) - seen_count_ids.add(counts_id) - - def _record_segment_safely( - self, - name: str, - start: float | None, - attrs: dict[str, Any], - *, - segment_tree_handle: dict[str, Any] | None = None, - ) -> None: - if start is None: - return - end = self.runtime._safe_perf_counter(f"segment.{name}.end") - if end is None: - return - try: - duration = end - start - self.runtime._finish_segment_tree_node( - segment_tree_handle, duration - ) - self.runtime._record_timing(name, duration) - context = self.runtime.current_context() - operation = self.runtime.current_operation() - metric_extra = { - key: value - for key, value in attrs.items() - if ( - key in self.runtime.config.metric_attribute_keys - and value is not None - ) - } - duration_ms = duration * 1000 - self.runtime._record_segment_flat_timing( - context, - operation, - name, - duration_ms, - ) - if operation is not None: - metric_attributes = operation.metric_attributes( - segment=name, - **metric_extra, - ) - elif context is not None: - metric_attributes = context.metric_attributes( - segment=name, - **metric_extra, - ) - else: - metric_attributes = _metric_attrs( - { - "service.name": self.runtime.config.service_name, - "service.role": self.runtime.config.service_role, - "deployment.environment": self.runtime.config.environment, - "segment": name, - **metric_extra, - }, - self.runtime.config.metric_attribute_keys, - ) - self.runtime.record_segment_metric( - name, - duration, - metric_attributes, - backend_segment="backend" in metric_extra, - ) - except BaseException as exc: - self.runtime.log_observability_failure( - "request.record_segment", - exc, - segment=name, - ) - - def _record_timing(self, name: str, duration_seconds: float) -> None: - try: - timings = _state._TIMINGS.get() - if timings is None: - return - key = f"{name}_ms" - duration_ms = duration_seconds * 1000.0 - timings[key] = round(timings.get(key, 0.0) + duration_ms, 1) - except BaseException as exc: - self.runtime.log_observability_failure( - "scope.record_timing", - exc, - segment=name, - ) - - def _coerce_segment(self, name: Any) -> str: - segment, is_registered = coerce_segment_name( - name, - registry=self.runtime.segment_registry, - ) - if not is_registered: - self.runtime.log_observability_failure( - "segment.coerce", - ValueError("Unregistered observability segment."), - segment=segment, - segment_type=type(name).__name__, - ) - return segment - - def _safe_perf_counter(self, operation: str) -> float | None: - try: - return time.perf_counter() - except BaseException as exc: - self.runtime.log_observability_failure(operation, exc) - return None - - -class _SegmentManager: - def __init__( - self, - runtime: ObservabilityRuntime, - name: Any, - attrs: dict[str, Any], - ) -> None: - self.runtime = runtime - self.name = name - self.attrs = attrs - self.context_manager = None - - def __enter__(self): - self.context_manager = self.runtime._segment_context( - self.name, - **self.attrs, - ) - return self.context_manager.__enter__() - - def __exit__(self, exc_type, exc, traceback) -> bool: - if self.context_manager is None: - return False - return bool(self.context_manager.__exit__(exc_type, exc, traceback)) - - async def __aenter__(self): - self.context_manager = self.runtime.asegment(self.name, **self.attrs) - return await self.context_manager.__aenter__() - - async def __aexit__(self, exc_type, exc, traceback) -> bool: - if self.context_manager is None: - return False - return bool( - await self.context_manager.__aexit__(exc_type, exc, traceback) - ) - - def __call__(self, func): - if inspect.iscoroutinefunction(func): - - @wraps(func) - async def async_wrapper(*args, **kwargs): - async with self.runtime.segment(self.name, **self.attrs): - return await func(*args, **kwargs) - - return async_wrapper - - @wraps(func) - def wrapper(*args, **kwargs): - with self.runtime.segment(self.name, **self.attrs): - return func(*args, **kwargs) - - return wrapper diff --git a/pyproject.toml b/pyproject.toml index 7818276..f92e9f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,23 +1,27 @@ [project] name = "policyengine-observability" -version = "1.4.1" +version = "2.0.0" description = "Shared PolicyEngine observability runtime for logs, timings, metrics, and OpenTelemetry." readme = "README.md" authors = [{ name = "PolicyEngine", email = "hello@policyengine.org" }] -requires-python = ">=3.12" -dependencies = [ +requires-python = ">=3.11" +dependencies = [] + +[project.optional-dependencies] +otel = [ "opentelemetry-api>=1.43.0", "opentelemetry-sdk>=1.43.0", +] +otlp-grpc = [ + "opentelemetry-api>=1.43.0", "opentelemetry-exporter-otlp-proto-grpc>=1.43.0", + "opentelemetry-sdk>=1.43.0", +] +otlp-http = [ + "opentelemetry-api>=1.43.0", "opentelemetry-exporter-otlp-proto-http>=1.43.0", - "opentelemetry-instrumentation-fastapi>=0.64b0", - "opentelemetry-instrumentation-httpx>=0.64b0", + "opentelemetry-sdk>=1.43.0", ] - -[project.optional-dependencies] -otel = [] -otlp-grpc = [] -otlp-http = [] flask = [ "flask>=2.2", ] @@ -28,18 +32,25 @@ httpx = [ "httpx", ] google = [ + "google-auth>=2.38.0", "google-cloud-logging>=3.15.0", ] all = [ "fastapi", "flask>=2.2", + "google-auth>=2.38.0", "google-cloud-logging>=3.15.0", "httpx", + "opentelemetry-api>=1.43.0", + "opentelemetry-exporter-otlp-proto-grpc>=1.43.0", + "opentelemetry-exporter-otlp-proto-http>=1.43.0", + "opentelemetry-sdk>=1.43.0", ] dev = [ "build", "coverage", "pytest", + "pyright>=1.1.405", "ruff>=0.9.0", "towncrier>=24.8.0", ] @@ -51,9 +62,19 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["policyengine_observability"] +[tool.hatch.build.targets.sdist] +exclude = [ + "/.claude", + "/.codex", + "/.pytest_cache", + "/.venv", + "/dist", + "/openspec", +] + [tool.ruff] line-length = 79 -target-version = "py312" +target-version = "py311" [tool.ruff.lint] select = [ @@ -76,6 +97,12 @@ line-ending = "auto" [tool.pytest.ini_options] testpaths = ["tests"] +[tool.pyright] +include = ["policyengine_observability"] +venvPath = "." +venv = ".venv" +pythonVersion = "3.11" + [tool.coverage.run] branch = true source = ["policyengine_observability"] diff --git a/tests/conftest.py b/tests/conftest.py index 9114b37..c19a707 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,41 +1,60 @@ from __future__ import annotations +import io +import json +from collections.abc import Iterator + import pytest -from fakes import RecordingDestination -from policyengine_observability import _state -from policyengine_observability.destinations.registry import ( - _STRATEGIES, - register_destination, +from policyengine_observability import ( + DeploymentIdentity, + LoggingConfig, + ObservabilityConfig, + OTelConfig, + ServiceIdentity, + configure, ) +from policyengine_observability.runtime import ObservabilityRuntime + + +def make_config(**overrides): + values = { + "service": ServiceIdentity( + name="test-api", + namespace="policyengine.test", + version="2.3.4", + role="entry", + ), + "deployment": DeploymentIdentity( + environment="test", + platform="local", + region="us-central1", + instance_id="instance-1", + ), + "logging": LoggingConfig(), + "otel": OTelConfig(enabled=False), + "application_attribute_keys": frozenset({"auth_result", "backend"}), + "dispatch_attribute_keys": frozenset( + {"job_id", "run_id", "simulation_id"} + ), + } + values.update(overrides) + return ObservabilityConfig(**values) + + +def make_runtime(**overrides) -> tuple[ObservabilityRuntime, io.StringIO]: + output = io.StringIO() + runtime = configure(make_config(**overrides)) + runtime._delivery._stdout = output + return runtime, output -@pytest.fixture(autouse=True) -def isolated_observability_context(): - """Keep tests independent when request and operation tests run in separate files.""" - variables = ( - (_state._REQUEST_CONTEXT, None), - (_state._OPERATION_CONTEXT, None), - (_state._TIMINGS, None), - (_state._TURN_START, None), - (_state._SEGMENT_STACK, ()), - ) - for variable, default in variables: - variable.set(default) - try: - yield - finally: - for variable, default in variables: - variable.set(default) +def records(output: io.StringIO) -> list[dict]: + return [json.loads(line) for line in output.getvalue().splitlines()] @pytest.fixture -def fake_remote_strategy(): - """Register a fake remote strategy; yields its destination name.""" - register_destination( - "fake-remote", - lambda **kwargs: RecordingDestination(), - transport="remote", - ) - yield "fake_remote" - _STRATEGIES.pop("fake_remote", None) +def runtime() -> Iterator[tuple[ObservabilityRuntime, io.StringIO]]: + value = make_runtime() + yield value + value[0].shutdown() diff --git a/tests/fakes.py b/tests/fakes.py deleted file mode 100644 index b4ea394..0000000 --- a/tests/fakes.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Shared destination fakes and manager builders for the test suite.""" - -from __future__ import annotations - -import json -import logging -import threading - -from policyengine_observability.config import ObservabilityConfig -from policyengine_observability.destinations.manager import ( - LogDestinationManager, -) - - -class RecordingLogger: - """Captures level-routed serialized lines like a stdlib logger.""" - - def __init__(self) -> None: - self.lines = [] - - def info(self, message) -> None: - self.lines.append(("INFO", message)) - - def warning(self, message) -> None: - self.lines.append(("WARNING", message)) - - def error(self, message) -> None: - self.lines.append(("ERROR", message)) - - -class RecordingDestination: - name = "recording" - - def __init__(self) -> None: - self.calls = [] - - @property - def payloads(self): - return [payload for payload, *_ in self.calls] - - def emit(self, payload, *, log_type, severity, timestamp=None) -> None: - self.calls.append((payload, log_type, severity, timestamp)) - - -class ClosableRecordingDestination(RecordingDestination): - """A recording destination with a zero-argument duck-typed close.""" - - def __init__(self) -> None: - super().__init__() - self.closed = 0 - - def close(self) -> None: - self.closed += 1 - - -class TimestampBlindDestination: - name = "timestamp-blind" - - def __init__(self) -> None: - self.calls = [] - - def emit(self, payload, *, log_type, severity) -> None: - self.calls.append((payload, log_type, severity)) - - -class BlockingDestination: - """Blocks inside emit until released; sequenced via Events.""" - - name = "blocking" - - def __init__(self) -> None: - self.entered = threading.Event() - self.release = threading.Event() - self.calls = 0 - self.closed = 0 - - def emit(self, payload, *, log_type, severity, timestamp=None) -> None: - self.calls += 1 - self.entered.set() - self.release.wait(10) - - def close(self) -> None: - self.closed += 1 - - -class FailingDestination: - """Raises from emit — always, or only for the first ``fail_first``.""" - - name = "failing" - - def __init__(self, fail_first: int | None = None) -> None: - self.calls = 0 - self.fail_first = fail_first - self.delivered = [] - - def emit(self, payload, *, log_type, severity, timestamp=None) -> None: - self.calls += 1 - if self.fail_first is None or self.calls <= self.fail_first: - raise RuntimeError("emit failed") - self.delivered.append(payload) - - -def make_manager(config=None, *, loggers=None, destinations=None): - """A manager with a failure-capturing on_failure. - - Returns ``(manager, failures)`` where each failure is the - ``(operation, exception, fields)`` triple the manager reported. - """ - failures = [] - manager = LogDestinationManager( - config=config or ObservabilityConfig(), - loggers=loggers or {"event": logging.getLogger("test-manager")}, - serializer=json.dumps, - on_failure=lambda operation, exc, **fields: failures.append( - (operation, exc, fields) - ), - ) - if destinations is not None: - manager.destinations = list(destinations) - manager.configured = True - return manager, failures diff --git a/tests/runtime_helpers.py b/tests/runtime_helpers.py deleted file mode 100644 index 5e287f5..0000000 --- a/tests/runtime_helpers.py +++ /dev/null @@ -1,198 +0,0 @@ -from __future__ import annotations - -from enum import StrEnum -from typing import Any - -from policyengine_observability import ( - ObservabilityConfig, - ObservabilityRuntime, -) - - -class SegmentName(StrEnum): - LOAD = "load" - SAVE = "save" - - -class RecordingSpan: - def __init__(self) -> None: - self.attributes = {} - self.exceptions = [] - self.events = [] - self.status = None - - def set_attribute(self, key, value) -> None: - self.attributes[key] = value - - def record_exception(self, exc) -> None: - self.exceptions.append(exc) - - def set_status(self, status) -> None: - self.status = status - - def add_event(self, event, fields) -> None: - self.events.append((event, fields)) - - def get_span_context(self): - return type( - "SpanContext", - (), - {"is_valid": False, "trace_id": 0, "span_id": 0}, - )() - - -class NamedRecordingSpan(RecordingSpan): - def __init__(self) -> None: - super().__init__() - self.names = [] - - def update_name(self, name: str) -> None: - self.names.append(name) - - -class ValidContextSpan(RecordingSpan): - def get_span_context(self): - return type( - "SpanContext", - (), - { - "is_valid": True, - "trace_id": 0x4BF92F3577B34DA6A3CE929D0E0E4736, - "span_id": 0x00F067AA0BA902B7, - }, - )() - - -class AttributeFailingSpan(RecordingSpan): - def set_attribute(self, key, value) -> None: - raise RuntimeError("attribute failed") - - -class ExceptionFailingSpan(RecordingSpan): - def record_exception(self, exc) -> None: - raise RuntimeError("record exception failed") - - -class RecordingSpanContextManager: - def __init__( - self, - span: RecordingSpan, - *, - fail_exit: bool = False, - ) -> None: - self.span = span - self.fail_exit = fail_exit - self.exited = False - - def __enter__(self): - return self.span - - def __exit__(self, *_args): - self.exited = True - if self.fail_exit: - raise RuntimeError("span exit failed") - return False - - -class RecordingTracer: - def __init__( - self, - span: RecordingSpan | None = None, - *, - fail_enter: bool = False, - fail_exit: bool = False, - ) -> None: - self.span = span or RecordingSpan() - self.fail_enter = fail_enter - self.fail_exit = fail_exit - self.calls = [] - self.last_context_manager = None - - def start_as_current_span(self, name, **kwargs): - self.calls.append((name, kwargs)) - if self.fail_enter: - raise RuntimeError("span enter failed") - self.last_context_manager = RecordingSpanContextManager( - self.span, - fail_exit=self.fail_exit, - ) - return self.last_context_manager - - -class RecordingMeter: - def __init__(self) -> None: - self.created = [] - - def create_histogram(self, name, **kwargs): - self.created.append(("histogram", name, kwargs)) - return RecordingInstrument() - - def create_counter(self, name, **kwargs): - self.created.append(("counter", name, kwargs)) - return RecordingInstrument() - - def create_up_down_counter(self, name, **kwargs): - self.created.append(("up_down_counter", name, kwargs)) - return RecordingInstrument() - - -class RecordingInstrument: - def __init__(self) -> None: - self.calls = [] - - def add(self, value, attributes=None) -> None: - self.calls.append(("add", value, attributes)) - - def record(self, value, attributes=None) -> None: - self.calls.append(("record", value, attributes)) - - -class FailingInstrument: - def add(self, *_args, **_kwargs) -> None: - raise RuntimeError("metric failed") - - def record(self, *_args, **_kwargs) -> None: - raise RuntimeError("metric failed") - - -class RecordingLogDestination: - def __init__(self, name: str = "recording") -> None: - self.name = name - self.calls = [] - - def emit( - self, - payload: dict[str, Any], - *, - log_type: str, - severity: str, - ) -> None: - self.calls.append((payload, log_type, severity)) - - -class FailingLogDestination: - name = "failing" - - def emit(self, *_args, **_kwargs) -> None: - raise RuntimeError("destination failed") - - -class RecordingPropagator: - def __init__(self) -> None: - self.extracted = None - - def inject(self, carrier) -> None: - carrier["traceparent"] = ( - "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" - ) - - def extract(self, carrier): - self.extracted = carrier - return {"parent": carrier} - - -def runtime(**kwargs) -> ObservabilityRuntime: - return ObservabilityRuntime( - ObservabilityConfig(service_name="svc", **kwargs), - segment_registry=SegmentName, - ) diff --git a/tests/test_adapters.py b/tests/test_adapters.py new file mode 100644 index 0000000..a157728 --- /dev/null +++ b/tests/test_adapters.py @@ -0,0 +1,308 @@ +from __future__ import annotations + +import asyncio +from typing import Any + +import httpx +import pytest +from conftest import make_runtime, records +from fastapi import FastAPI +from fastapi.testclient import TestClient +from flask import Flask +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.trace import StatusCode + +from policyengine_observability import ( + REQUEST_ID_HEADER, + OTelConfig, + instrument_fastapi, + instrument_flask, + instrument_httpx, +) + + +def test_flask_lifecycle_and_idempotence() -> None: + runtime, output = make_runtime() + unused, _ = make_runtime() + app = Flask(__name__) + + @app.get("/items/") + def item(item_id: str): + return {"item": item_id} + + assert instrument_flask(app, runtime) is runtime + assert instrument_flask(app, unused) is runtime + response = app.test_client().get( + "/items/abc", headers={REQUEST_ID_HEADER: "flask-request"} + ) + assert response.status_code == 200 + assert response.headers[REQUEST_ID_HEADER] == "flask-request" + emitted = records(output) + assert len(emitted) == 1 + assert emitted[0]["http.route"] == "/items/" + runtime.shutdown() + unused.shutdown() + + +def test_flask_exception_emits_one_error_completion() -> None: + runtime, output = make_runtime(otel=OTelConfig(enabled=True)) + exporter = InMemorySpanExporter() + runtime._otel._tracer_provider.add_span_processor( + SimpleSpanProcessor(exporter) + ) + app = Flask(__name__) + + @app.get("/fail") + def fail(): + raise ValueError("route failed") + + instrument_flask(app, runtime) + response = app.test_client().get("/fail") + assert response.status_code == 500 + emitted = records(output) + assert len(emitted) == 1 + assert emitted[0]["outcome"] == "error" + assert emitted[0]["error.type"] == "ValueError" + assert emitted[0]["error.message"] == "route failed" + span = exporter.get_finished_spans()[0] + assert span.status.status_code is StatusCode.ERROR + assert span.events[0].name == "exception" + assert span.events[0].attributes["exception.message"] == "route failed" + runtime.shutdown() + + +def test_flask_observes_early_before_request_response() -> None: + runtime, output = make_runtime() + app = Flask(__name__) + + @app.before_request + def reject_request(): + return {"error": "unauthorized"}, 401 + + instrument_flask(app, runtime) + response = app.test_client().get("/protected") + + assert response.status_code == 401 + assert REQUEST_ID_HEADER in response.headers + emitted = records(output) + assert len(emitted) == 1 + assert emitted[0]["http.response.status_code"] == 401 + assert emitted[0]["outcome"] == "client_error" + runtime.shutdown() + + +def test_flask_late_installation_is_nonfatal_and_leaves_no_state() -> None: + runtime, output = make_runtime() + app = Flask(__name__) + + @app.get("/health") + def health(): + return {"status": "ok"} + + client = app.test_client() + assert client.get("/health").status_code == 200 + callbacks_before = _flask_callback_snapshot(app) + + assert instrument_flask(app, runtime) is runtime + + assert _flask_callback_snapshot(app) == callbacks_before + assert "policyengine_observability" not in app.extensions + assert runtime.diagnostics.count("failure.flask.callback_install") == 1 + assert client.get("/health").status_code == 200 + assert records(output) == [] + runtime.shutdown() + + +@pytest.mark.parametrize( + "registration_name", + ("before_request", "after_request", "teardown_request"), +) +def test_flask_installation_failure_rolls_back_and_can_retry( + monkeypatch, registration_name +) -> None: + runtime, output = make_runtime() + app = Flask(__name__) + + @app.get("/health") + def health(): + return {"status": "ok"} + + callbacks_before = _flask_callback_snapshot(app) + original = getattr(app, registration_name) + + def register_then_fail(callback): + original(callback) + raise RuntimeError("registration failed") + + monkeypatch.setattr(app, registration_name, register_then_fail) + + assert instrument_flask(app, runtime) is runtime + assert _flask_callback_snapshot(app) == callbacks_before + assert "policyengine_observability" not in app.extensions + assert runtime.diagnostics.count("failure.flask.callback_install") == 1 + + monkeypatch.setattr(app, registration_name, original) + assert instrument_flask(app, runtime) is runtime + response = app.test_client().get("/health") + + assert response.status_code == 200 + assert REQUEST_ID_HEADER in response.headers + assert len(records(output)) == 1 + runtime.shutdown() + + +def test_fastapi_lifecycle_and_idempotence() -> None: + runtime, output = make_runtime() + unused, _ = make_runtime() + app = FastAPI() + + @app.get("/items/{item_id}") + async def item(item_id: str): + return {"item": item_id} + + assert instrument_fastapi(app, runtime) is runtime + assert instrument_fastapi(app, unused) is runtime + with TestClient(app) as client: + response = client.get( + "/items/abc", headers={REQUEST_ID_HEADER: "fastapi-request"} + ) + assert response.status_code == 200 + assert response.headers[REQUEST_ID_HEADER] == "fastapi-request" + emitted = records(output) + assert len(emitted) == 1 + assert emitted[0]["http.route"] == "/items/{item_id}" + runtime.shutdown() + unused.shutdown() + + +def test_fastapi_response_observability_failure_preserves_response( + monkeypatch, +) -> None: + runtime, output = make_runtime() + app = FastAPI() + + @app.get("/ok") + async def ok(): + return {"status": "ok"} + + def fail_response_headers(): + raise RuntimeError("response instrumentation failed") + + monkeypatch.setattr(runtime, "response_headers", fail_response_headers) + instrument_fastapi(app, runtime) + + with TestClient(app, raise_server_exceptions=False) as client: + response = client.get("/ok") + + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + assert REQUEST_ID_HEADER not in response.headers + assert runtime.diagnostics.count("failure.fastapi.response_headers") == 1 + assert records(output)[0]["outcome"] == "success" + runtime.shutdown() + + +def test_fastapi_error_and_non_http_scope() -> None: + runtime, output = make_runtime() + app = FastAPI() + + @app.get("/fail") + async def fail(): + raise ValueError("route failed") + + instrument_fastapi(app, runtime) + with TestClient(app, raise_server_exceptions=False) as client: + response = client.get("/fail") + assert response.status_code == 500 + emitted = records(output) + assert len(emitted) == 1 + assert emitted[0]["outcome"] == "error" + runtime.shutdown() + + +def test_httpx_only_instruments_supplied_sync_client() -> None: + runtime, _output = make_runtime() + seen: list[httpx.Request] = [] + + def respond(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(200) + + supplied = httpx.Client(transport=httpx.MockTransport(respond)) + untouched = httpx.Client(transport=httpx.MockTransport(respond)) + assert instrument_httpx(supplied, runtime) is supplied + instrument_httpx(supplied, runtime) + runtime.begin_request( + headers={REQUEST_ID_HEADER: "outbound-request"}, + method="GET", + route="/dispatch", + ) + supplied.get("https://example.test/supplied") + untouched.get("https://example.test/untouched") + assert seen[0].headers[REQUEST_ID_HEADER] == "outbound-request" + assert REQUEST_ID_HEADER not in seen[1].headers + supplied.close() + untouched.close() + runtime.end_request(status_code=200) + runtime.shutdown() + + +def test_httpx_async_client_and_invalid_client_are_nonfatal() -> None: + runtime, _output = make_runtime() + seen: list[httpx.Request] = [] + + async def respond(request: httpx.Request) -> httpx.Response: + seen.append(request) + return httpx.Response(200) + + async def run() -> None: + client = httpx.AsyncClient(transport=httpx.MockTransport(respond)) + instrument_httpx(client, runtime) + runtime.begin_request( + headers={REQUEST_ID_HEADER: "async-request"}, + method="POST", + route="/dispatch", + ) + await client.get("https://example.test/async") + await client.aclose() + runtime.end_request(status_code=200) + + asyncio.run(run()) + assert seen[0].headers[REQUEST_ID_HEADER] == "async-request" + marker = object() + assert instrument_httpx(marker, runtime) is marker + assert runtime.diagnostics.count("failure.httpx.instrument") == 1 + runtime.shutdown() + + +def test_httpx_client_cannot_be_rebound_to_another_runtime() -> None: + first, _ = make_runtime() + second, _ = make_runtime() + client = httpx.Client( + transport=httpx.MockTransport(lambda _: httpx.Response(200)) + ) + instrument_httpx(client, first) + instrument_httpx(client, second) + assert second.diagnostics.count("failure.httpx.already_instrumented") == 1 + client.close() + first.shutdown() + second.shutdown() + + +def _flask_callback_snapshot( + app: Flask, +) -> dict[str, dict[Any, tuple[Any, ...]]]: + return { + name: { + key: tuple(callbacks) + for key, callbacks in getattr(app, name).items() + } + for name in ( + "before_request_funcs", + "after_request_funcs", + "teardown_request_funcs", + ) + } diff --git a/tests/test_config_schema.py b/tests/test_config_schema.py new file mode 100644 index 0000000..5a5c699 --- /dev/null +++ b/tests/test_config_schema.py @@ -0,0 +1,483 @@ +from __future__ import annotations + +import json + +import pytest +from conftest import make_config + +from policyengine_observability import ( + SCHEMA_VERSION, + ConfigurationError, + CustomLogDestination, + DeploymentIdentity, + GoogleCloudLogDestination, + GoogleCloudLogFormatter, + LoggingConfig, + ObservabilityConfig, + OTelConfig, + OTLPExporterConfig, + ServiceIdentity, + StdoutLogDestination, + TelemetryLimits, + configure, +) +from policyengine_observability.schema import ( + build_record, + metric_attributes, + normalize_attributes, +) + + +def test_explicit_identity_is_required_by_constructor() -> None: + config = make_config() + assert config.identity_complete + assert config.service.namespace == "policyengine.test" + assert config.deployment.platform == "local" + + +def test_invalid_configuration_fails_before_destination_setup() -> None: + destination_was_built = False + + def build_writer(): + nonlocal destination_was_built + destination_was_built = True + + config = make_config( + service=ServiceIdentity("", "", "", ""), + deployment=DeploymentIdentity("", "invalid"), # type: ignore[arg-type] + logging=LoggingConfig( + destinations=( + GoogleCloudLogDestination(project_id="", log_name=""), + CustomLogDestination( + name="unbuilt", + writer_factory=build_writer, + delivery="inline", + ), + ), + minimum_severity="INVALID", # type: ignore[arg-type] + ), + ) + + with pytest.raises(ConfigurationError) as raised: + configure(config) + + message = str(raised.value) + assert "service.name must be a non-empty string" in message + assert "deployment.platform" in message + assert "logging.minimum_severity" in message + assert "Google Cloud logging requires project_id" in message + assert not destination_was_built + + +def test_missing_otel_exporters_are_a_nonfatal_diagnostic() -> None: + config = make_config(otel=OTelConfig(enabled=True)) + + config.validate() + + assert "no trace or metric OTLP endpoint" in " ".join(config.diagnostics()) + + +def test_invalid_nested_limits_are_reported_together() -> None: + exporter = OTLPExporterConfig( + endpoint="", + protocol="invalid", # type: ignore[arg-type] + endpoint_mode="invalid", # type: ignore[arg-type] + timeout_seconds=0, + ) + config = make_config( + logging=LoggingConfig(shutdown_timeout_seconds=float("nan")), + otel=OTelConfig( + traces=exporter, + provider_mode="invalid", # type: ignore[arg-type] + sampling_ratio=2, + span_queue_capacity=0, + span_batch_size=0, + span_schedule_delay_seconds=0, + metric_export_interval_seconds=0, + shutdown_timeout_seconds=100, + ), + limits=TelemetryLimits( + max_attributes=0, + max_string_length=0, + max_error_message_length=0, + max_stack_length=0, + async_parent_max_age_seconds=-1, + ), + ) + + message = " ".join(config.validation_errors()) + + for field in ( + "logging.shutdown_timeout_seconds", + "otel.provider_mode", + "otel.sampling_ratio", + "otel.span_queue_capacity", + "otel.span_batch_size", + "otel.span_schedule_delay_seconds", + "otel.metric_export_interval_seconds", + "otel.shutdown_timeout_seconds", + "otel.traces.endpoint", + "otel.traces.protocol", + "otel.traces.endpoint_mode", + "otel.traces.timeout_seconds", + "limits.max_attributes", + "limits.async_parent_max_age_seconds", + ): + assert field in message + + +@pytest.mark.parametrize( + ("sensitive_values", "expected"), + [ + (["secret"], "sensitive_values must be a tuple"), + ((123,), r"sensitive_values\[0\] must be a non-empty string"), + (("",), r"sensitive_values\[0\] must be a non-empty string"), + ((" ",), r"sensitive_values\[0\] must be a non-empty string"), + ], +) +def test_invalid_sensitive_values_fail_before_runtime_setup( + sensitive_values, expected +) -> None: + config = make_config( + sensitive_values=sensitive_values, # type: ignore[arg-type] + ) + + with pytest.raises(ConfigurationError, match=expected): + configure(config) + + +@pytest.mark.parametrize( + ("field_name", "value"), + [ + ("application_attribute_keys", ("backend",)), + ("application_attribute_keys", frozenset({""})), + ("dispatch_attribute_keys", ["job_id"]), + ("dispatch_attribute_keys", frozenset({1})), + ("metric_attribute_keys", {"outcome"}), + ("metric_attribute_keys", frozenset({" "})), + ], +) +def test_invalid_attribute_allowlists_fail_before_runtime_setup( + field_name, value +) -> None: + config = make_config(**{field_name: value}) + + with pytest.raises(ConfigurationError, match=field_name): + configure(config) + + +def test_invalid_destination_strategies_are_reported() -> None: + class InvalidDestination: + name = "invalid" + delivery = "network" + queue_capacity = 0 + batch_size = 0 + + def diagnostics(self): + raise RuntimeError("validation failed") + + config = make_config( + logging=LoggingConfig( + destinations=( + object(), # type: ignore[arg-type] + InvalidDestination(), # type: ignore[arg-type] + CustomLogDestination( + name="", + writer_factory=None, # type: ignore[arg-type] + formatter=object(), # type: ignore[arg-type] + ), + StdoutLogDestination(formatter=object()), # type: ignore[arg-type] + GoogleCloudLogDestination( + project_id="", + log_name="", + write_timeout_seconds=0, + ), + ) + ) + ) + + message = " ".join(config.validation_errors()) + + for expected in ( + "Invalid log destination strategy", + "delivery must be one of", + "queue_capacity", + "batch_size", + "validation failed", + "name must be non-empty", + "writer_factory must be callable", + "formatter must be callable", + "requires project_id", + "requires log_name", + "write_timeout_seconds", + ): + assert expected in message + + +def test_remote_logging_is_not_selected_from_platform() -> None: + config = make_config( + logging=LoggingConfig( + destinations=( + GoogleCloudLogDestination( + project_id="central", log_name="application" + ), + ) + ) + ) + assert config.diagnostics() == () + + +def test_from_env_reads_transport_but_not_identity(monkeypatch) -> None: + monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "ambient-project") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://collector") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "x-one=1,x-two=2") + monkeypatch.setenv("OTEL_TRACES_SAMPLER_ARG", "0.25") + monkeypatch.setenv("OTEL_BSP_MAX_QUEUE_SIZE", "12") + monkeypatch.setenv("OTEL_BSP_MAX_EXPORT_BATCH_SIZE", "99") + monkeypatch.setenv("OTEL_BSP_SCHEDULE_DELAY", "2500") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TIMEOUT", "1500") + monkeypatch.setenv("OTEL_METRIC_EXPORT_INTERVAL", "4000") + service = ServiceIdentity("svc", "ns", "1", "api") + deployment = DeploymentIdentity("prod", "google_cloud_run") + config = ObservabilityConfig.from_env( + service=service, deployment=deployment + ) + assert config.service is service + assert config.deployment is deployment + assert config.otel.traces == OTLPExporterConfig( + endpoint="https://collector", + protocol="http/protobuf", + headers=(("x-one", "1"), ("x-two", "2")), + timeout_seconds=1.5, + ) + assert config.otel.metrics == config.otel.traces + assert config.otel.sampling_ratio == 0.25 + assert config.otel.span_queue_capacity == 12 + assert config.otel.span_batch_size == 99 + assert config.otel.span_schedule_delay_seconds == 2.5 + assert config.otel.metric_export_interval_seconds == 4.0 + + +def test_from_env_marks_signal_specific_endpoints_as_exact( + monkeypatch, +) -> None: + monkeypatch.setenv( + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "https://collector/custom-traces", + ) + monkeypatch.setenv( + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + "https://collector/custom-metrics", + ) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf") + + config = ObservabilityConfig.from_env( + service=ServiceIdentity("svc", "ns", "1", "api"), + deployment=DeploymentIdentity("dev", "local"), + ) + + assert config.otel.traces is not None + assert config.otel.metrics is not None + assert config.otel.traces.endpoint_mode == "signal" + assert config.otel.metrics.endpoint_mode == "signal" + + +@pytest.mark.parametrize( + ("name", "value"), + [ + ("OTEL_EXPORTER_OTLP_PROTOCOL", "invalid"), + ("OTEL_TRACES_SAMPLER_ARG", "nan"), + ("OTEL_BSP_MAX_QUEUE_SIZE", "bad"), + ("POLICYENGINE_OTEL_PROVIDER_MODE", "invalid"), + ("OTEL_TRACES_EXPORTER", "console"), + ("OTEL_SDK_DISABLED", "sometimes"), + ], +) +def test_from_env_rejects_invalid_values(monkeypatch, name, value) -> None: + monkeypatch.setenv(name, value) + + with pytest.raises(ConfigurationError, match=name): + ObservabilityConfig.from_env( + service=ServiceIdentity("svc", "ns", "1", "api"), + deployment=DeploymentIdentity("dev", "local"), + ) + + +def test_from_env_rejects_malformed_headers(monkeypatch) -> None: + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "collector:4317") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "missing-equals") + + with pytest.raises(ConfigurationError, match="without '='"): + ObservabilityConfig.from_env( + service=ServiceIdentity("svc", "ns", "1", "api"), + deployment=DeploymentIdentity("dev", "local"), + ) + + +def test_from_env_accepts_false_boolean_and_disabled_exporters( + monkeypatch, +) -> None: + monkeypatch.setenv("OTEL_SDK_DISABLED", "no") + monkeypatch.setenv("OTEL_TRACES_EXPORTER", "none") + monkeypatch.setenv("OTEL_METRICS_EXPORTER", "none") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "collector:4317") + + config = ObservabilityConfig.from_env( + service=ServiceIdentity("svc", "ns", "1", "api"), + deployment=DeploymentIdentity("dev", "local"), + ) + + assert config.otel.enabled + assert config.otel.traces is None + assert config.otel.metrics is None + + +@pytest.mark.parametrize( + ("name", "value", "message"), + [ + ( + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", + "invalid", + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", + ), + ("OTEL_EXPORTER_OTLP_HEADERS", "=value", "empty header name"), + ("OTEL_EXPORTER_OTLP_TIMEOUT", "50", "must be between"), + ("OTEL_BSP_MAX_QUEUE_SIZE", "0", "must be between"), + ], +) +def test_from_env_rejects_invalid_exporter_values( + monkeypatch, name, value, message +) -> None: + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "collector:4317") + monkeypatch.setenv(name, value) + + with pytest.raises(ConfigurationError, match=message): + ObservabilityConfig.from_env( + service=ServiceIdentity("svc", "ns", "1", "api"), + deployment=DeploymentIdentity("dev", "local"), + ) + + +def test_schema_preserves_core_fields_and_namespaces_attributes() -> None: + config = make_config( + application_attribute_keys=frozenset({"service.name", "backend"}) + ) + record = build_record( + config, + severity="info", + event_name="work.started", + context={"request.id": "request-1"}, + attributes={"service.name": "attacker", "backend": "modal"}, + ) + assert record["schema_version"] == SCHEMA_VERSION + assert record["service.name"] == "test-api" + assert record["attributes"]["service.name"] == "attacker" + assert record["attributes"]["backend"] == "modal" + json.dumps(record) + + +def test_schema_redacts_and_truncates_errors() -> None: + config = make_config( + application_attribute_keys=frozenset({"backend"}), + sensitive_values=("secret-value",), + limits=TelemetryLimits( + max_string_length=64, + max_error_message_length=64, + max_stack_length=100, + ), + ) + try: + raise ValueError("secret-value should disappear") + except ValueError as error: + record = build_record( + config, + severity="ERROR", + message="secret-value message", + attributes={"backend": "prefix-secret-value-suffix"}, + error=error, + ) + assert "secret-value" not in str(record) + assert "[REDACTED]" in str(record) + assert record["attributes"]["backend"] == "prefix-[REDACTED]-suffix" + assert len(record["error.stack"]) <= 100 + + +def test_schema_redacts_before_truncating_strings() -> None: + config = make_config( + application_attribute_keys=frozenset({"backend"}), + sensitive_values=("secret-value",), + limits=TelemetryLimits(max_string_length=8), + ) + + record = build_record( + config, + severity="INFO", + message="secret-value", + attributes={"backend": "secret-value"}, + ) + + assert record["message"] == "[REDACTE" + assert record["attributes"]["backend"] == "[REDACTE" + + +def test_attribute_policy_omits_sensitive_non_scalar_and_nonfinite() -> None: + config = make_config( + application_attribute_keys=frozenset( + {"allowed", "authorization", "items", "infinite", "flag"} + ) + ) + safe, omitted = normalize_attributes( + { + "allowed": "value", + "authorization": "bearer", + "items": [1, 2], + "infinite": float("inf"), + "flag": True, + }, + config, + allowed_keys=config.application_attribute_keys, + ) + assert safe == {"allowed": "value", "flag": True} + assert omitted == 3 + + +def test_attribute_count_and_string_length_are_bounded() -> None: + config = make_config( + application_attribute_keys=frozenset({"one", "two", "three"}), + limits=TelemetryLimits(max_attributes=2, max_string_length=3), + ) + safe, omitted = normalize_attributes( + {"one": "abcdef", "two": 2, "three": 3}, config + ) + assert safe == {"one": "abc", "two": 2} + assert omitted == 1 + + +def test_google_trace_correlation_is_not_in_canonical_record() -> None: + record = build_record( + make_config(), + severity="INFO", + event_name="correlated", + context={ + "trace_id": "a" * 32, + "span_id": "b" * 16, + "trace_sampled": True, + }, + ) + assert "logging.googleapis.com/trace" not in record + assert "logging.googleapis.com/spanId" not in record + assert "logging.googleapis.com/trace_sampled" not in record + formatted = GoogleCloudLogFormatter("central-project")(record) + assert formatted["logging.googleapis.com/trace"] == ( + "projects/central-project/traces/" + "a" * 32 + ) + assert formatted["logging.googleapis.com/spanId"] == "b" * 16 + assert formatted["logging.googleapis.com/trace_sampled"] is True + + +def test_metric_attributes_use_separate_allowlist() -> None: + config = make_config() + assert metric_attributes( + {"service.name": "test-api", "job_id": "high-cardinality"}, config + ) == {"service.name": "test-api"} diff --git a/tests/test_delivery.py b/tests/test_delivery.py new file mode 100644 index 0000000..dca6368 --- /dev/null +++ b/tests/test_delivery.py @@ -0,0 +1,388 @@ +from __future__ import annotations + +import io +import json +import threading +import time + +from conftest import make_config + +from policyengine_observability import ( + CustomLogDestination, + DeploymentIdentity, + GoogleCloudLogDestination, + LoggingConfig, + StdoutLogDestination, +) +from policyengine_observability.delivery import DeliveryManager +from policyengine_observability.destinations.google_cloud import ( + _GoogleCloudWriter, +) +from policyengine_observability.diagnostics import Diagnostics + + +class RecordingWriter: + def __init__(self) -> None: + self.records: list[dict] = [] + self.closed = False + self.written = threading.Event() + + def write(self, record: dict) -> None: + self.records.append(record) + self.written.set() + + def write_many(self, records: list[dict]) -> None: + self.records.extend(records) + self.written.set() + + def close(self) -> None: + self.closed = True + + +def queued_config(writer_factory, **overrides): + queue_capacity = overrides.pop("queue_capacity", 10) + batch_size = overrides.pop("batch_size", 5) + shutdown_timeout_seconds = overrides.pop("shutdown_timeout_seconds", 0.2) + platform = overrides.pop("platform", "modal") + assert not overrides + return make_config( + deployment=DeploymentIdentity("test", platform), + logging=LoggingConfig( + destinations=( + StdoutLogDestination(), + CustomLogDestination( + name="recording", + writer_factory=writer_factory, + delivery="queued", + queue_capacity=queue_capacity, + batch_size=batch_size, + ), + ), + shutdown_timeout_seconds=shutdown_timeout_seconds, + ), + ) + + +def test_stdout_delivery_is_single_line_json() -> None: + output = io.StringIO() + manager = DeliveryManager(make_config(), Diagnostics(), stdout=output) + manager.emit({"severity": "INFO", "event.name": "test"}) + assert output.getvalue().count("\n") == 1 + assert json.loads(output.getvalue())["event.name"] == "test" + assert not manager.remote_enabled + + +def test_queued_delivery_writes_stdout_and_uses_lazy_worker() -> None: + output = io.StringIO() + diagnostics = Diagnostics() + writer = RecordingWriter() + factory_calls = 0 + + def factory(): + nonlocal factory_calls + factory_calls += 1 + return writer + + manager = DeliveryManager( + queued_config(factory), + diagnostics, + stdout=output, + ) + assert factory_calls == 0 + manager.emit({"severity": "INFO", "event.name": "queued"}) + assert json.loads(output.getvalue())["event.name"] == "queued" + assert writer.written.wait(1) + assert writer.records[0]["event.name"] == "queued" + assert factory_calls == 1 + manager.close(1) + assert writer.closed + + +def test_explicit_queued_destination_is_platform_independent() -> None: + writer = RecordingWriter() + manager = DeliveryManager( + queued_config(lambda: writer, platform="google_cloud_run"), + Diagnostics(), + stdout=io.StringIO(), + ) + manager.emit({"event.name": "cloud-run"}) + assert writer.written.wait(1) + manager.close(1) + + +def test_queue_saturation_drops_without_blocking() -> None: + started = threading.Event() + release = threading.Event() + diagnostics = Diagnostics() + + class BlockingWriter(RecordingWriter): + def write_many(self, records: list[dict]) -> None: + started.set() + release.wait(1) + super().write_many(records) + + writer = BlockingWriter() + manager = DeliveryManager( + queued_config( + lambda: writer, + queue_capacity=1, + batch_size=1, + ), + diagnostics, + stdout=io.StringIO(), + ) + manager.emit({"sequence": 1}) + assert started.wait(1) + manager.emit({"sequence": 2}) + before = time.perf_counter() + manager.emit({"sequence": 3}) + assert time.perf_counter() - before < 0.1 + assert diagnostics.count("logs.dropped.queue_full") == 1 + release.set() + manager.close(1) + + +def test_writer_auth_or_permission_failure_is_nonfatal() -> None: + diagnostics = Diagnostics() + attempted = threading.Event() + + def unavailable(): + attempted.set() + raise PermissionError("denied") + + manager = DeliveryManager( + queued_config(unavailable), + diagnostics, + stdout=io.StringIO(), + ) + manager.emit({"event.name": "survives"}) + assert attempted.wait(1) + deadline = time.time() + 1 + while diagnostics.count("logs.export_failure") == 0: + assert time.time() < deadline + time.sleep(0.01) + manager.close(1) + assert diagnostics.count("logs.export_failure") == 1 + + +def test_malformed_record_and_closed_queue_are_nonfatal() -> None: + diagnostics = Diagnostics() + manager = DeliveryManager(make_config(), diagnostics, stdout=io.StringIO()) + manager.emit({"bad": object()}) + assert diagnostics.count("logs.export_failure") == 1 + + remote = DeliveryManager( + queued_config(lambda: RecordingWriter()), + diagnostics, + stdout=io.StringIO(), + ) + remote.close(1) + remote.emit({"event.name": "after-close"}) + assert diagnostics.count("logs.dropped.closed") == 1 + + +def test_shutdown_timeout_is_bounded_and_repeatable() -> None: + started = threading.Event() + release = threading.Event() + diagnostics = Diagnostics() + + class BlockingWriter(RecordingWriter): + def write_many(self, records: list[dict]) -> None: + started.set() + release.wait(1) + + manager = DeliveryManager( + queued_config( + lambda: BlockingWriter(), + shutdown_timeout_seconds=0.01, + ), + diagnostics, + stdout=io.StringIO(), + ) + manager.emit({"event.name": "block"}) + assert started.wait(1) + before = time.perf_counter() + manager.close(0.01) + manager.close(0.01) + assert time.perf_counter() - before < 0.2 + assert diagnostics.count("logs.shutdown_timeout") == 1 + release.set() + + +def test_inline_failure_cleanup_does_not_block_emit() -> None: + close_started = threading.Event() + release = threading.Event() + + class FailingWriter: + def write(self, _record: dict) -> None: + raise RuntimeError("write failed") + + def close(self) -> None: + close_started.set() + release.wait(1) + + manager = DeliveryManager( + make_config( + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="failing-inline", + writer_factory=FailingWriter, + delivery="inline", + ), + ), + ) + ), + Diagnostics(stderr=io.StringIO()), + stdout=io.StringIO(), + ) + + manager.emit({"sequence": 1}) + manager.emit({"sequence": 2}) + before = time.perf_counter() + manager.emit({"sequence": 3}) + + assert time.perf_counter() - before < 0.1 + assert close_started.wait(1) + release.set() + manager.close(1) + + +def test_inline_shutdown_cleanup_is_bounded_and_repeatable() -> None: + close_started = threading.Event() + release = threading.Event() + diagnostics = Diagnostics(stderr=io.StringIO()) + close_calls = 0 + + class BlockingCloseWriter(RecordingWriter): + def close(self) -> None: + nonlocal close_calls + close_calls += 1 + close_started.set() + release.wait(1) + + manager = DeliveryManager( + make_config( + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="blocking-inline", + writer_factory=BlockingCloseWriter, + delivery="inline", + ), + ), + shutdown_timeout_seconds=0.01, + ) + ), + diagnostics, + stdout=io.StringIO(), + ) + + before = time.perf_counter() + manager.close(0.01) + manager.close(0.01) + + assert time.perf_counter() - before < 0.2 + assert close_started.is_set() + assert close_calls == 1 + assert diagnostics.count("logs.shutdown_timeout") == 1 + release.set() + + +def test_inline_cleanup_failure_is_contained() -> None: + close_attempted = threading.Event() + diagnostics = Diagnostics(stderr=io.StringIO()) + + class FailingCloseWriter(RecordingWriter): + def close(self) -> None: + close_attempted.set() + raise RuntimeError("close failed") + + manager = DeliveryManager( + make_config( + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="failing-close", + writer_factory=FailingCloseWriter, + delivery="inline", + ), + ), + ) + ), + diagnostics, + stdout=io.StringIO(), + ) + + manager.close(1) + + assert close_attempted.is_set() + assert diagnostics.count("failure.logging.writer_close") == 1 + + +def test_google_writer_batches_with_bounded_api_calls(monkeypatch) -> None: + from google.cloud import logging_v2 + + committed: list[dict] = [] + + class Batch: + def log_struct(self, record, **kwargs): + committed.append({"record": record, "kwargs": kwargs}) + + def commit(self): + committed.append({"committed": True}) + + class Logger: + def batch(self): + return Batch() + + class Gapic: + def write_log_entries(self, *args, **kwargs): + return None + + class Client: + def __init__(self, **kwargs): + self.kwargs = kwargs + self.logging_api = type("API", (), {"_gapic_api": Gapic()})() + self.closed = False + + def logger(self, name): + assert name == "application" + return Logger() + + def close(self): + self.closed = True + + monkeypatch.setattr(logging_v2, "Client", Client) + monkeypatch.setattr( + "policyengine_observability.google_credentials.load_google_credentials", + lambda **_kwargs: "credentials", + ) + writer = _GoogleCloudWriter( + GoogleCloudLogDestination( + project_id="central", + log_name="application", + write_timeout_seconds=2, + ) + ) + writer.write_many( + [ + { + "severity": "WARNING", + "trace_id": "a", + "span_id": "b", + "trace_sampled": True, + }, + {"severity": "INFO"}, + ] + ) + assert committed[-1] == {"committed": True} + assert committed[0]["kwargs"] == { + "severity": "WARNING", + "trace": "projects/central/traces/a", + "span_id": "b", + "trace_sampled": True, + } + assert committed[1]["kwargs"]["trace_sampled"] is False + writer.close() + assert writer._client.closed diff --git a/tests/test_destination_strategies.py b/tests/test_destination_strategies.py new file mode 100644 index 0000000..5371b24 --- /dev/null +++ b/tests/test_destination_strategies.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +import io +import json +import threading + +from conftest import make_config + +from policyengine_observability import ( + CustomLogDestination, + DeploymentIdentity, + GoogleCloudLogFormatter, + LoggingConfig, + StdoutLogDestination, +) +from policyengine_observability.delivery import DeliveryManager +from policyengine_observability.destinations import DestinationBuildContext +from policyengine_observability.diagnostics import Diagnostics + + +class RecordingWriter: + def __init__(self) -> None: + self.records: list[dict] = [] + self.written = threading.Event() + + def write(self, record: dict) -> None: + self.records.append(record) + self.written.set() + + def close(self) -> None: + pass + + +class FailingWriter: + def write(self, _record: dict) -> None: + raise RuntimeError("destination failed") + + +def test_remote_destination_selection_is_independent_of_platform() -> None: + writer = RecordingWriter() + manager = DeliveryManager( + make_config( + deployment=DeploymentIdentity("test", "other"), + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="recording", + writer_factory=lambda: writer, + delivery="queued", + ), + ) + ), + ), + Diagnostics(), + ) + + manager.emit({"event.name": "portable"}) + + assert writer.written.wait(1) + assert writer.records == [{"event.name": "portable"}] + manager.close(1) + + +def test_multiple_runtimes_can_write_to_separate_destinations() -> None: + first = RecordingWriter() + second = RecordingWriter() + + def manager_for(writer: RecordingWriter) -> DeliveryManager: + return DeliveryManager( + make_config( + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="isolated", + writer_factory=lambda: writer, + delivery="queued", + ), + ) + ) + ), + Diagnostics(), + ) + + first_manager = manager_for(first) + second_manager = manager_for(second) + first_manager.emit({"consumer": "first-api"}) + second_manager.emit({"consumer": "household-api"}) + + assert first.written.wait(1) + assert second.written.wait(1) + assert first.records == [{"consumer": "first-api"}] + assert second.records == [{"consumer": "household-api"}] + first_manager.close(1) + second_manager.close(1) + + +def test_google_fields_are_added_by_destination_formatter() -> None: + output = io.StringIO() + manager = DeliveryManager( + make_config( + logging=LoggingConfig( + destinations=( + StdoutLogDestination( + formatter=GoogleCloudLogFormatter("trace-project") + ), + ) + ) + ), + Diagnostics(), + stdout=output, + ) + + manager.emit( + { + "severity": "INFO", + "trace_id": "a" * 32, + "span_id": "b" * 16, + "trace_sampled": True, + } + ) + + record = json.loads(output.getvalue()) + assert record["logging.googleapis.com/trace"] == ( + "projects/trace-project/traces/" + "a" * 32 + ) + assert record["logging.googleapis.com/spanId"] == "b" * 16 + assert record["logging.googleapis.com/trace_sampled"] is True + + +def test_one_destination_failure_does_not_affect_another_destination() -> None: + successful = RecordingWriter() + diagnostics = Diagnostics() + manager = DeliveryManager( + make_config( + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="failing", + writer_factory=FailingWriter, + delivery="inline", + ), + CustomLogDestination( + name="successful", + writer_factory=lambda: successful, + delivery="inline", + ), + ) + ) + ), + diagnostics, + ) + + for sequence in range(4): + manager.emit({"sequence": sequence}) + + assert successful.records == [ + {"sequence": 0}, + {"sequence": 1}, + {"sequence": 2}, + {"sequence": 3}, + ] + assert diagnostics.count("logs.export_failure") == 3 + assert diagnostics.count("failure.logging.destination_disabled") == 1 + + +def test_formatters_receive_deeply_isolated_records() -> None: + formatted = RecordingWriter() + unchanged = RecordingWriter() + source = {"attributes": {"backend": "original"}} + + def mutate_nested(record: dict) -> dict: + record["attributes"]["backend"] = "formatted" + return record + + manager = DeliveryManager( + make_config( + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="formatted", + writer_factory=lambda: formatted, + delivery="inline", + formatter=mutate_nested, + ), + CustomLogDestination( + name="unchanged", + writer_factory=lambda: unchanged, + delivery="inline", + ), + ) + ) + ), + Diagnostics(), + ) + + manager.emit(source) + + assert formatted.records[0]["attributes"]["backend"] == "formatted" + assert unchanged.records[0]["attributes"]["backend"] == "original" + assert source["attributes"]["backend"] == "original" + manager.close(1) + + +def test_destination_writers_copy_nested_values_before_formatting() -> None: + source = {"attributes": {"backend": "original"}} + + def mutate_nested(record: dict) -> dict: + record["attributes"]["backend"] = "formatted" + return record + + output = io.StringIO() + context = DestinationBuildContext(stdout=lambda: output) + stdout_writer = StdoutLogDestination(formatter=mutate_nested).build_writer( + context + ) + stdout_writer.write(source) + + assert json.loads(output.getvalue())["attributes"]["backend"] == ( + "formatted" + ) + assert source["attributes"]["backend"] == "original" + + recording = RecordingWriter() + custom_writer = CustomLogDestination( + name="custom", + writer_factory=lambda: recording, + delivery="inline", + formatter=mutate_nested, + ).build_writer(context) + custom_writer.write(source) + + assert recording.records[0]["attributes"]["backend"] == "formatted" + assert source["attributes"]["backend"] == "original" + + +def test_slow_remote_destination_does_not_delay_another_destination() -> None: + started = threading.Event() + release = threading.Event() + successful = RecordingWriter() + + class BlockingWriter: + def write(self, _record: dict) -> None: + started.set() + release.wait(1) + + manager = DeliveryManager( + make_config( + logging=LoggingConfig( + destinations=( + CustomLogDestination( + name="blocking", + writer_factory=BlockingWriter, + ), + CustomLogDestination( + name="successful", + writer_factory=lambda: successful, + ), + ) + ) + ), + Diagnostics(), + ) + + manager.emit({"event.name": "independent"}) + + assert started.wait(1) + assert successful.written.wait(0.2) + release.set() + manager.close(1) diff --git a/tests/test_diagnostics_public_api.py b/tests/test_diagnostics_public_api.py new file mode 100644 index 0000000..f7412b0 --- /dev/null +++ b/tests/test_diagnostics_public_api.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import ast +import io +import json +import logging +import sys + +import pytest +from conftest import make_config, make_runtime, records + +import policyengine_observability as observability +from policyengine_observability.diagnostics import Diagnostics + +REMOVED_NAMES = { + "segment", + "asegment", + "entrypoint", + "set_attribute", + "record_event", + "record_error", + "collect_timings", + "start_scope", + "annotate", + "end_scope", + "set_observability_runtime", + "register_destination", + "GoogleCloudLoggingConfig", +} + + +def test_public_api_contains_only_version_two_surface() -> None: + assert REMOVED_NAMES.isdisjoint(observability.__all__) + assert all(not hasattr(observability, name) for name in REMOVED_NAMES) + assert ( + observability.configure(make_config()).config.service.name + == "test-api" + ) + + +def test_import_does_not_require_framework_google_or_otel_sdk_modules() -> ( + None +): + package_source = open(observability.__file__).read() + tree = ast.parse(package_source) + top_level_modules = { + alias.name.split(".")[0] + for node in tree.body + if isinstance(node, ast.Import) + for alias in node.names + } + assert top_level_modules.isdisjoint( + {"google", "opentelemetry", "flask", "fastapi", "httpx"} + ) + + +def test_diagnostics_are_local_rate_limited_and_counted() -> None: + output = io.StringIO() + diagnostics = Diagnostics(stderr=output, interval_seconds=60) + diagnostics.report("export.failed", ValueError("one"), attempt=1) + diagnostics.report("export.failed", ValueError("two"), attempt=2) + diagnostics.increment("dropped", 3) + lines = output.getvalue().splitlines() + assert len(lines) == 1 + item = json.loads(lines[0]) + assert item["operation"] == "export.failed" + assert item["attempt"] == 1 + assert diagnostics.count("failure.export.failed") == 2 + assert diagnostics.count("dropped") == 3 + + +def test_diagnostics_redact_configured_sensitive_values() -> None: + output = io.StringIO() + diagnostics = Diagnostics( + stderr=output, + sensitive_values=("secret-value",), + ) + + diagnostics.report( + "export.failed", + ValueError("secret-value exporter failure"), + destination="prefix-secret-value-suffix", + ) + + rendered = output.getvalue() + item = json.loads(rendered) + assert "secret-value" not in rendered + assert item["error.message"] == "[REDACTED] exporter failure" + assert item["destination"] == "prefix-[REDACTED]-suffix" + + +def test_diagnostic_listener_failure_does_not_escape() -> None: + diagnostics = Diagnostics(stderr=io.StringIO()) + diagnostics.add_listener( + lambda _name, _value: (_ for _ in ()).throw(RuntimeError("listener")) + ) + diagnostics.increment("still-counted", 2) + assert diagnostics.count("still-counted") == 2 + + +def test_standard_logging_ignores_observability_dependencies() -> None: + runtime, output = make_runtime() + logger = logging.getLogger("opentelemetry.exporter") + logger.handlers.clear() + logger.propagate = False + observability.instrument_logging(logger, runtime) + logger.error("would recurse") + assert records(output) == [] + runtime.shutdown() + + +def test_automatic_standard_logging_installation(monkeypatch) -> None: + logger = logging.getLogger() + original_handlers = list(logger.handlers) + try: + config = make_config( + logging=observability.LoggingConfig( + capture_standard_library=True, + replace_existing_handlers=True, + ) + ) + runtime = observability.configure(config) + runtime._delivery._stdout = io.StringIO() + logging.getLogger("application.auto").warning("automatic") + assert records(runtime._delivery._stdout)[0]["message"] == "automatic" + runtime.shutdown() + finally: + logger.handlers[:] = original_handlers + + +def test_invalid_automatic_logging_config_preserves_existing_handlers() -> ( + None +): + logger = logging.getLogger() + original_handlers = list(logger.handlers) + existing = logging.StreamHandler(io.StringIO()) + logger.handlers[:] = [existing] + try: + config = make_config( + logging=observability.LoggingConfig( + capture_standard_library=True, + replace_existing_handlers=True, + minimum_severity="INVALID", # type: ignore[arg-type] + ) + ) + + with pytest.raises( + observability.ConfigurationError, + match="logging.minimum_severity", + ): + observability.configure(config) + + assert logger.handlers == [existing] + finally: + logger.handlers[:] = original_handlers + + +def test_automatic_logging_installation_failure_is_nonfatal( + monkeypatch, +) -> None: + logger = logging.getLogger() + original_handlers = list(logger.handlers) + existing = logging.StreamHandler(io.StringIO()) + logger.handlers[:] = [existing] + + def fail_add_handler(_handler) -> None: + raise RuntimeError("logging subsystem rejected handler") + + try: + with monkeypatch.context() as logging_failure: + logging_failure.setattr(logger, "addHandler", fail_add_handler) + runtime = observability.configure( + make_config( + logging=observability.LoggingConfig( + capture_standard_library=True, + replace_existing_handlers=True, + ) + ) + ) + + assert logger.handlers == [existing] + assert ( + runtime.diagnostics.count("failure.logging.handler_install") == 1 + ) + runtime.shutdown() + finally: + logger.handlers[:] = original_handlers + + +def test_replace_logging_handlers_and_shutdown_removes_owned_handler() -> None: + runtime, _ = make_runtime() + logger = logging.getLogger("tests.replace") + logger.handlers.clear() + logger.addHandler(logging.StreamHandler(sys.stderr)) + handler = observability.instrument_logging(logger, runtime, replace=True) + assert logger.handlers == [handler] + runtime.shutdown() + assert logger.handlers == [] diff --git a/tests/test_fastapi_adapter.py b/tests/test_fastapi_adapter.py deleted file mode 100644 index 1ff9055..0000000 --- a/tests/test_fastapi_adapter.py +++ /dev/null @@ -1,380 +0,0 @@ -from __future__ import annotations - -import asyncio -from enum import StrEnum - -import pytest - -from policyengine_observability import ( - ObservabilityConfig, - ObservabilityRuntime, -) -from policyengine_observability.adapters.fastapi import ( - UNMATCHED_ROUTE, - FastAPIObservabilityAdapter, - FastAPIObservabilityMiddleware, - _endpoint_from_scope, - _headers_from_scope, - _int_header, - _merge_response_headers, - _query_keys, - _route_from_scope, - _split_forwarded_for, - init_fastapi_observability, -) - - -class SegmentName(StrEnum): - LOAD = "load" - - -class RecordingInstrument: - def __init__(self) -> None: - self.calls = [] - - def add(self, value, attributes=None) -> None: - self.calls.append(("add", value, attributes)) - - def record(self, value, attributes=None) -> None: - self.calls.append(("record", value, attributes)) - - -def _fastapi_modules(): - fastapi = pytest.importorskip("fastapi") - responses = pytest.importorskip("starlette.responses") - return fastapi, responses - - -def _call_asgi(app, path: str): - async def run(): - messages = [] - received_request = False - scope = { - "type": "http", - "asgi": {"version": "3.0", "spec_version": "2.4"}, - "http_version": "1.1", - "method": "GET", - "scheme": "http", - "path": path, - "raw_path": path.encode(), - "query_string": b"", - "headers": [(b"host", b"testserver")], - "client": ("127.0.0.1", 12345), - "server": ("testserver", 80), - "root_path": "", - } - - async def receive(): - nonlocal received_request - if not received_request: - received_request = True - return { - "type": "http.request", - "body": b"", - "more_body": False, - } - await asyncio.sleep(60) - return {"type": "http.disconnect"} - - async def send(message): - messages.append(message) - - await app(scope, receive, send) - return messages - - return asyncio.run(run()) - - -def test_fastapi_streaming_request_finishes_after_final_body() -> None: - fastapi, responses = _fastapi_modules() - app = fastapi.FastAPI() - observed = ObservabilityRuntime( - ObservabilityConfig( - service_name="svc", - otel_enabled=False, - metric_attribute_keys=( - "service.name", - "route", - "method", - "segment", - "tool", - ), - ), - segment_registry=SegmentName, - ) - observed.requests = RecordingInstrument() - observed.http_duration = RecordingInstrument() - observed.segment_duration = RecordingInstrument() - observed.active_requests = RecordingInstrument() - - @app.get("/items/{item_id}") - async def get_item(item_id: str): - async def body(): - with observed.segment(SegmentName.LOAD, tool="stream"): - yield f"item:{item_id}".encode() - - return responses.StreamingResponse(body()) - - init_fastapi_observability( - app, - runtime=observed, - service_name="svc", - ) - - messages = _call_asgi(app, "/items/abc") - response_start = next( - message - for message in messages - if message["type"] == "http.response.start" - ) - body = b"".join( - message.get("body", b"") - for message in messages - if message["type"] == "http.response.body" - ) - response_headers = dict(response_start["headers"]) - - assert body == b"item:abc" - assert response_headers[b"X-PolicyEngine-Request-Id"] - _, _, request_attributes = observed.requests.calls[0] - _, _, segment_attributes = observed.segment_duration.calls[0] - assert request_attributes["route"] == "/items/{item_id}" - assert segment_attributes["route"] == "/items/{item_id}" - assert segment_attributes["tool"] == "stream" - assert observed.active_requests.calls[0][1] == 1 - assert observed.active_requests.calls[-1][1] == -1 - - -def test_fastapi_unmatched_route_uses_stable_metric_label() -> None: - fastapi, _responses = _fastapi_modules() - app = fastapi.FastAPI() - observed = ObservabilityRuntime( - ObservabilityConfig(service_name="svc", otel_enabled=False) - ) - observed.requests = RecordingInstrument() - observed.http_duration = RecordingInstrument() - observed.active_requests = RecordingInstrument() - - init_fastapi_observability( - app, - runtime=observed, - service_name="svc", - ) - - messages = _call_asgi(app, "/missing/abc") - response_start = next( - message - for message in messages - if message["type"] == "http.response.start" - ) - - assert response_start["status"] == 404 - _, _, request_attributes = observed.requests.calls[0] - assert request_attributes["route"] == UNMATCHED_ROUTE - - -def test_fastapi_static_attributes_are_recorded() -> None: - fastapi, _responses = _fastapi_modules() - app = fastapi.FastAPI() - observed = ObservabilityRuntime( - ObservabilityConfig( - service_name="svc", - otel_enabled=False, - metric_attribute_keys=( - "service.name", - "route", - "method", - "platform", - "runtime_role", - ), - ) - ) - observed.requests = RecordingInstrument() - observed.http_duration = RecordingInstrument() - observed.active_requests = RecordingInstrument() - - @app.get("/ok") - async def ok(): - return {"ok": True} - - init_fastapi_observability( - app, - runtime=observed, - service_name="svc", - static_attributes={ - "platform": "modal", - "runtime_role": "modal_web", - "ignored": None, - }, - ) - - _call_asgi(app, "/ok") - - _, _, request_attributes = observed.requests.calls[0] - assert request_attributes["platform"] == "modal" - assert request_attributes["runtime_role"] == "modal_web" - assert "ignored" not in request_attributes - - -def test_fastapi_adapter_disabled_and_idempotent_paths() -> None: - fastapi, _responses = _fastapi_modules() - app = fastapi.FastAPI() - disabled = ObservabilityRuntime.disabled() - adapter = FastAPIObservabilityAdapter(disabled) - - adapter.instrument_app(app) - - assert not hasattr(app.state, "policyengine_observability_adapter") - - observed = ObservabilityRuntime( - ObservabilityConfig(service_name="svc", instrument_fastapi=True) - ) - adapter = FastAPIObservabilityAdapter(observed) - adapter.instrument_app(app) - adapter.instrument_app(app) - - assert app.state.policyengine_observability_adapter is adapter - - -def test_fastapi_adapter_logs_middleware_and_start_failures() -> None: - class BrokenApp: - state = type("State", (), {})() - - def add_middleware(self, *_args, **_kwargs): - raise RuntimeError("middleware failed") - - observed = ObservabilityRuntime(ObservabilityConfig(service_name="svc")) - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - adapter = FastAPIObservabilityAdapter(observed) - adapter.instrument_app(BrokenApp()) - observed.begin_request = lambda *_args, **_kwargs: (_ for _ in ()).throw( - RuntimeError("begin failed") - ) - adapter.start_request({"headers": [], "path": "/broken"}) - - assert failures == [ - "fastapi.middleware_install", - "fastapi.before_request", - ] - - -def test_fastapi_init_returns_existing_runtime() -> None: - fastapi, _responses = _fastapi_modules() - app = fastapi.FastAPI() - runtime = ObservabilityRuntime.disabled() - app.state.policyengine_observability = runtime - - assert init_fastapi_observability(app, service_name="svc") is runtime - - -def test_fastapi_inbound_metadata_ip_source_variants() -> None: - observed = ObservabilityRuntime( - ObservabilityConfig(service_name="svc", log_raw_ip=False) - ) - adapter = FastAPIObservabilityAdapter(observed) - - real_ip = adapter._inbound_metadata( - {"client": ("10.0.0.1", 123)}, - {"x-real-ip": "198.51.100.5"}, - ) - remote_addr = adapter._inbound_metadata( - {"client": ("10.0.0.1", 123)}, - {}, - ) - - assert real_ip["ip_source"] == "x_real_ip" - assert remote_addr["ip_source"] == "remote_addr" - assert "client_ip" not in real_ip - - -def test_fastapi_middleware_non_http_scope_passthrough() -> None: - calls = [] - - async def app(scope, _receive, _send): - calls.append(scope["type"]) - - adapter = FastAPIObservabilityAdapter( - ObservabilityRuntime(ObservabilityConfig(service_name="svc")) - ) - middleware = FastAPIObservabilityMiddleware(app, adapter=adapter) - - async def run(): - await middleware({"type": "lifespan"}, None, None) - - asyncio.run(run()) - - assert calls == ["lifespan"] - - -def test_fastapi_middleware_records_exception_before_reraising() -> None: - observed = ObservabilityRuntime(ObservabilityConfig(service_name="svc")) - observed.errors = RecordingInstrument() - observed.requests = RecordingInstrument() - observed.http_duration = RecordingInstrument() - observed.active_requests = RecordingInstrument() - - async def app(_scope, _receive, _send): - raise RuntimeError("app failed") - - adapter = FastAPIObservabilityAdapter(observed) - middleware = FastAPIObservabilityMiddleware(app, adapter=adapter) - - async def run(): - async def receive(): - return {"type": "http.request", "body": b"", "more_body": False} - - async def send(_message): - return None - - scope = { - "type": "http", - "method": "GET", - "path": "/boom", - "query_string": b"a=1", - "headers": [], - "client": ("127.0.0.1", 123), - } - with pytest.raises(RuntimeError, match="app failed"): - await middleware(scope, receive, send) - - asyncio.run(run()) - - assert observed.errors.calls[0][0] == "add" - assert observed.requests.calls[0][0] == "add" - - -def test_fastapi_helpers_handle_edge_inputs() -> None: - scope = { - "headers": [ - (b"x-test", b"one"), - (b"x-test", b"two"), - (object(), object()), - ], - "query_string": "a=1&b=&a=2", - } - - assert _headers_from_scope(scope)["x-test"] == "one,two" - assert _query_keys(scope) == ["a", "b"] - assert _route_from_scope({"route": object()}) is None - assert _endpoint_from_scope({"endpoint": "callable-ish"}) == "callable-ish" - assert _int_header("bad") is None - assert _int_header("12") == 12 - assert _split_forwarded_for("1.1.1.1, ,2.2.2.2") == [ - "1.1.1.1", - "2.2.2.2", - ] - assert _merge_response_headers( - [(b"x-policyengine-request-id", b"old"), (b"x-other", b"keep")], - { - "X-PolicyEngine-Request-Id": "new", - "traceparent": "parent", - "ignored": "value", - }, - ) == [ - (b"x-other", b"keep"), - (b"X-PolicyEngine-Request-Id", b"new"), - (b"traceparent", b"parent"), - ] diff --git a/tests/test_flask_adapter.py b/tests/test_flask_adapter.py deleted file mode 100644 index 7397746..0000000 --- a/tests/test_flask_adapter.py +++ /dev/null @@ -1,171 +0,0 @@ -from __future__ import annotations - -import pytest - -from policyengine_observability import ( - ObservabilityConfig, - ObservabilityRuntime, -) -from policyengine_observability.adapters.flask import ( - FlaskObservabilityAdapter, - _split_forwarded_for, - init_flask_observability, -) - - -class RecordingInstrument: - def __init__(self) -> None: - self.calls = [] - - def add(self, value, attributes=None) -> None: - self.calls.append(("add", value, attributes)) - - def record(self, value, attributes=None) -> None: - self.calls.append(("record", value, attributes)) - - -def _flask(): - return pytest.importorskip("flask") - - -def test_flask_adapter_records_request_metadata_and_headers() -> None: - flask = _flask() - app = flask.Flask(__name__) - runtime = ObservabilityRuntime( - ObservabilityConfig( - service_name="svc", - otel_enabled=False, - metric_attribute_keys=( - "service.name", - "route", - "method", - "status_code", - "segment", - "tool", - ), - ) - ) - runtime.requests = RecordingInstrument() - runtime.http_duration = RecordingInstrument() - runtime.active_requests = RecordingInstrument() - runtime.segment_duration = RecordingInstrument() - - @app.get("/items/") - def item(item_id: str): - runtime.set_attribute("tool", "handler") - with runtime.segment("load", tool="handler"): - pass - return {"item_id": item_id} - - initialized = init_flask_observability( - app, - runtime=runtime, - service_name="svc", - ) - - response = app.test_client().get( - "/items/abc?debug=1", - headers={ - "X-Forwarded-For": "203.0.113.1, 198.51.100.2", - "User-Agent": "pytest", - "Origin": "https://policyengine.org", - }, - ) - - assert initialized is runtime - assert response.status_code == 200 - assert response.headers["X-PolicyEngine-Request-Id"] - _, _, request_attributes = runtime.requests.calls[0] - _, _, segment_attributes = runtime.segment_duration.calls[0] - assert request_attributes["route"] == "/items/" - assert request_attributes["status_code"] == "200" - assert segment_attributes["tool"] == "handler" - assert runtime.active_requests.calls[0][1] == 1 - assert runtime.active_requests.calls[-1][1] == -1 - - -def test_flask_adapter_is_idempotent_and_honors_disabled_runtime() -> None: - flask = _flask() - app = flask.Flask(__name__) - runtime = ObservabilityRuntime.disabled() - - first = init_flask_observability( - app, - runtime=runtime, - service_name="svc", - ) - second = init_flask_observability( - app, - runtime=ObservabilityRuntime( - ObservabilityConfig(service_name="other") - ), - service_name="other", - ) - - assert first is runtime - assert second is runtime - assert "policyengine_observability_adapter" not in app.extensions - - -def test_flask_adapter_enabled_idempotence_and_start_failure() -> None: - flask = _flask() - app = flask.Flask(__name__) - runtime = ObservabilityRuntime(ObservabilityConfig(service_name="svc")) - adapter = FlaskObservabilityAdapter(runtime) - failures = [] - runtime.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - - adapter.instrument_app(app) - adapter.instrument_app(app) - adapter.start_request() - - assert app.extensions["policyengine_observability_adapter"] is adapter - assert failures == ["flask.before_request"] - - -def test_flask_init_builds_runtime_from_config_and_returns_existing() -> None: - flask = _flask() - app = flask.Flask(__name__) - - first = init_flask_observability( - app, - config=ObservabilityConfig(service_name="svc", enabled=False), - service_name="ignored", - ) - second = init_flask_observability(app, service_name="other") - - assert first is second - assert first.config.service_name == "svc" - - -def test_flask_inbound_metadata_ip_source_variants() -> None: - flask = _flask() - app = flask.Flask(__name__) - runtime = ObservabilityRuntime( - ObservabilityConfig(service_name="svc", log_raw_ip=False) - ) - adapter = FlaskObservabilityAdapter(runtime) - - with app.test_request_context( - "/", - headers={"X-Real-IP": "198.51.100.5"}, - ): - metadata = adapter._inbound_metadata(flask.request) - assert metadata["ip_source"] == "x_real_ip" - assert "client_ip" not in metadata - - with app.test_request_context( - "/", environ_base={"REMOTE_ADDR": "10.0.0.1"} - ): - metadata = adapter._inbound_metadata(flask.request) - assert metadata["ip_source"] == "remote_addr" - - -def test_split_forwarded_for_discards_empty_parts() -> None: - assert _split_forwarded_for(None) == [] - assert _split_forwarded_for(" 1.1.1.1, ,2.2.2.2 ") == [ - "1.1.1.1", - "2.2.2.2", - ] diff --git a/tests/test_google_credentials.py b/tests/test_google_credentials.py index 4870c26..29a137b 100644 --- a/tests/test_google_credentials.py +++ b/tests/test_google_credentials.py @@ -1,323 +1,106 @@ from __future__ import annotations import json -import os -import stat -import tempfile -import pytest +from policyengine_observability import google_credentials -from policyengine_observability.destinations import ( - google_cloud_logging, - google_credentials, -) -from policyengine_observability.destinations.google_credentials import ( - configure_google_application_credentials, - load_google_credentials, -) - -@pytest.fixture(autouse=True) -def clear_google_credential_env(monkeypatch) -> None: - for key in ( +def test_missing_and_malformed_credentials_are_nonfatal(monkeypatch) -> None: + for name in ( "GCP_CREDENTIALS_JSON", "GOOGLE_APPLICATION_CREDENTIALS", - "MODAL_IDENTITY_TOKEN", - "OBSERVABILITY_GOOGLE_OIDC_TOKEN", - "OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL", - "OBSERVABILITY_GOOGLE_STS_TOKEN_URL", - "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER", + google_credentials.MODAL_IDENTITY_TOKEN_ENV, + google_credentials.OIDC_TOKEN_ENV, + google_credentials.WORKLOAD_IDENTITY_PROVIDER_ENV, ): - monkeypatch.delenv(key, raising=False) - - -def test_configure_google_application_credentials_preserves_existing_env( - monkeypatch, -) -> None: - monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", "/existing.json") - monkeypatch.setenv("GCP_CREDENTIALS_JSON", '{"project_id":"test"}') - - path = configure_google_application_credentials() - - assert str(path) == "/existing.json" - assert os.environ["GOOGLE_APPLICATION_CREDENTIALS"] == "/existing.json" - - -def test_configure_google_application_credentials_noops_without_json( - monkeypatch, -) -> None: - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) - monkeypatch.delenv("GCP_CREDENTIALS_JSON", raising=False) - - assert configure_google_application_credentials() is None - assert "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ - - -def test_configure_google_application_credentials_materializes_json( - monkeypatch, - tmp_path, -) -> None: - credentials_path = tmp_path / "credentials.json" - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) - monkeypatch.setenv("GCP_CREDENTIALS_JSON", '{"project_id":"test"}') - - path = configure_google_application_credentials( - credentials_path=credentials_path, - ) - - assert path == credentials_path - assert credentials_path.read_text() == '{"project_id":"test"}' - assert stat.S_IMODE(credentials_path.stat().st_mode) == 0o600 - assert os.environ["GOOGLE_APPLICATION_CREDENTIALS"] == str( - credentials_path - ) - - -def test_configure_google_application_credentials_rejects_invalid_json( - monkeypatch, - tmp_path, -) -> None: - credentials_path = tmp_path / "credentials.json" - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) + monkeypatch.delenv(name, raising=False) + assert google_credentials.load_google_credentials() is None monkeypatch.setenv("GCP_CREDENTIALS_JSON", "not-json") - - path = configure_google_application_credentials( - credentials_path=credentials_path, - ) - - assert path is None - assert not credentials_path.exists() - assert "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ - - -def test_configure_google_application_credentials_fails_open_on_unexpected_error( - monkeypatch, - tmp_path, -) -> None: - credentials_path = tmp_path / "credentials.json" - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) - monkeypatch.setenv("GCP_CREDENTIALS_JSON", '{"project_id":"test"}') - monkeypatch.setattr( - "policyengine_observability.destinations.google_credentials.json.loads", - lambda _value: (_ for _ in ()).throw(RuntimeError("boom")), - ) - - path = configure_google_application_credentials( - credentials_path=credentials_path, - ) - - assert path is None - assert not credentials_path.exists() - assert "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ + assert google_credentials.load_google_credentials() is None -def test_configure_google_application_credentials_fails_open_on_write_error( - monkeypatch, - tmp_path, -) -> None: - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) - monkeypatch.setenv("GCP_CREDENTIALS_JSON", '{"project_id":"test"}') +def test_json_credentials_are_loaded_in_memory(monkeypatch) -> None: + import google.auth - path = configure_google_application_credentials( - credentials_path=tmp_path / "missing" / "credentials.json", - ) - - assert path is None - assert "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ - - -def test_configure_google_application_credentials_materializes_oidc_wif( - monkeypatch, - tmp_path, -) -> None: - monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path)) - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) - monkeypatch.delenv("GCP_CREDENTIALS_JSON", raising=False) - monkeypatch.setenv("OBSERVABILITY_GOOGLE_OIDC_TOKEN", "jwt-token") + config = {"type": "external_account", "audience": "test"} monkeypatch.setenv( - "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER", - "projects/123/locations/global/workloadIdentityPools/modal/providers/modal", - ) - monkeypatch.setenv( - "OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL", - "observability-writer@example.iam.gserviceaccount.com", + "GCP_CREDENTIALS_JSON", + json.dumps(config), ) + calls: dict[str, object] = {} + marker = object() - path = configure_google_application_credentials() - - assert path == tmp_path / "policyengine-observability-wif.json" - assert os.environ["GOOGLE_APPLICATION_CREDENTIALS"] == str(path) - assert stat.S_IMODE(path.stat().st_mode) == 0o600 - token_path = tmp_path / "policyengine-observability-oidc.jwt" - assert token_path.read_text() == "jwt-token" - assert stat.S_IMODE(token_path.stat().st_mode) == 0o600 - config = json.loads(path.read_text()) - assert config == { - "type": "external_account", - "audience": ( - "//iam.googleapis.com/projects/123/locations/global/" - "workloadIdentityPools/modal/providers/modal" - ), - "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", - "token_url": "https://sts.googleapis.com/v1/token", - "credential_source": { - "file": str(token_path), - "format": {"type": "text"}, - }, - "service_account_impersonation_url": ( - "https://iamcredentials.googleapis.com/v1/projects/-/" - "serviceAccounts/observability-writer@example.iam.gserviceaccount.com" - ":generateAccessToken" - ), - } + def load_credentials_from_dict(info, *, scopes): + calls["info"] = info + calls["scopes"] = scopes + return marker, "project" - -def test_configure_google_application_credentials_preserves_full_wif_audience( - monkeypatch, - tmp_path, -) -> None: - monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path)) - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) - monkeypatch.delenv("GCP_CREDENTIALS_JSON", raising=False) - monkeypatch.setenv("OBSERVABILITY_GOOGLE_OIDC_TOKEN", "jwt-token") - monkeypatch.setenv( - "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER", - "//iam.googleapis.com/projects/123/locations/global/" - "workloadIdentityPools/modal/providers/modal", + monkeypatch.setattr( + google.auth, + "load_credentials_from_dict", + load_credentials_from_dict, ) + assert google_credentials.load_google_credentials() is marker + assert calls["info"] == config + assert calls["scopes"] == list(google_credentials.GOOGLE_CREDENTIAL_SCOPES) - path = configure_google_application_credentials() - - config = json.loads(path.read_text()) - assert config["audience"] == ( - "//iam.googleapis.com/projects/123/locations/global/" - "workloadIdentityPools/modal/providers/modal" - ) - assert "service_account_impersonation_url" not in config +def test_modal_workload_identity_configuration(monkeypatch) -> None: + from google.auth import identity_pool -def test_configure_google_application_credentials_uses_modal_identity_token( - monkeypatch, - tmp_path, -) -> None: - monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path)) - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) - monkeypatch.delenv("GCP_CREDENTIALS_JSON", raising=False) - monkeypatch.delenv("OBSERVABILITY_GOOGLE_OIDC_TOKEN", raising=False) - monkeypatch.setenv("MODAL_IDENTITY_TOKEN", "modal-jwt-token") monkeypatch.setenv( - "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER", - "projects/123/locations/global/workloadIdentityPools/modal/providers/modal", + google_credentials.MODAL_IDENTITY_TOKEN_ENV, "signed-test-token" ) - - path = configure_google_application_credentials() - - assert path == tmp_path / "policyengine-observability-wif.json" - token_path = tmp_path / "policyengine-observability-oidc.jwt" - assert token_path.read_text() == "modal-jwt-token" - - -def test_load_google_credentials_prefers_wif_without_mutating_adc( - monkeypatch, - tmp_path, -) -> None: - monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path)) - monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", "/analytics.json") - monkeypatch.setenv("GCP_CREDENTIALS_JSON", '{"project_id":"analytics"}') - monkeypatch.setenv("MODAL_IDENTITY_TOKEN", "modal-jwt-token") monkeypatch.setenv( - "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER", - "projects/123/locations/global/workloadIdentityPools/modal/providers/modal", - ) - calls = [] - monkeypatch.setattr( - google_credentials, - "_load_credentials_from_file", - lambda path: calls.append(path) or "wif-credentials", + google_credentials.WORKLOAD_IDENTITY_PROVIDER_ENV, + "projects/123/locations/global/workloadIdentityPools/modal/providers/example", ) - - credentials = load_google_credentials(prefer_workload_identity=True) - - assert credentials == "wif-credentials" - assert calls == [tmp_path / "policyengine-observability-wif.json"] - assert os.environ["GOOGLE_APPLICATION_CREDENTIALS"] == "/analytics.json" - - -def test_load_google_credentials_loads_identity_pool_credentials( - monkeypatch, - tmp_path, -) -> None: - from google.auth import identity_pool - - monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path)) - monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) - monkeypatch.delenv("GCP_CREDENTIALS_JSON", raising=False) - monkeypatch.setenv("MODAL_IDENTITY_TOKEN", "modal-jwt-token") monkeypatch.setenv( - "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER", - "projects/123/locations/global/workloadIdentityPools/modal/providers/modal", + google_credentials.SERVICE_ACCOUNT_EMAIL_ENV, + "modal-example@central.iam.gserviceaccount.com", ) + calls: dict[str, object] = {} - credentials = load_google_credentials(prefer_workload_identity=True) - - assert isinstance(credentials, identity_pool.Credentials) - assert "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ + def credentials(**kwargs): + calls.update(kwargs) + return "credentials" - -def test_google_destination_bootstraps_application_credentials( - monkeypatch, -) -> None: - calls = [] - monkeypatch.setattr( - google_cloud_logging, - "configure_google_application_credentials", - lambda: calls.append("configured"), + monkeypatch.setattr(identity_pool, "Credentials", credentials) + config = google_credentials.load_google_credentials( + prefer_workload_identity=True ) - monkeypatch.setattr( - google_cloud_logging, - "load_google_credentials", - lambda *, prefer_workload_identity: None, + assert config == "credentials" + assert str(calls["audience"]).startswith( + "//iam.googleapis.com/projects/123" ) - - class FakeClient: - project = "test-project" - - def logger(self, log_name): - return log_name - - destination = google_cloud_logging.GoogleCloudLoggingDestination( - project=None, - log_name="policyengine-observability", - client_factory=lambda _project, _credentials: FakeClient(), + assert "modal-example@central.iam.gserviceaccount.com" in str( + calls["service_account_impersonation_url"] ) + assert "credential_source" not in calls + supplier = calls["subject_token_supplier"] + assert supplier.get_subject_token(None, None) == "signed-test-token" - assert calls == ["configured"] - assert destination.project == "test-project" - assert destination.logger == "policyengine-observability" - -def test_google_destination_passes_loaded_credentials(monkeypatch) -> None: +def test_existing_credentials_path_is_loaded(monkeypatch, tmp_path) -> None: + path = tmp_path / "adc.json" + path.write_text("{}") + monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", str(path)) + marker = object() monkeypatch.setattr( - google_cloud_logging, - "load_google_credentials", - lambda *, prefer_workload_identity: "wif-credentials", + google_credentials, + "_load_credentials_from_file", + lambda loaded: marker if loaded == path else None, ) + assert google_credentials.load_google_credentials() is marker - calls = [] - - class FakeClient: - project = "test-project" - def logger(self, log_name): - return log_name - - destination = google_cloud_logging.GoogleCloudLoggingDestination( - project="central-project", - log_name="policyengine-observability", - client_factory=lambda project, credentials: ( - calls.append((project, credentials)) or FakeClient() - ), +def test_workload_identity_audience_preserves_supported_forms() -> None: + full = "//iam.googleapis.com/projects/123/providers/test" + assert google_credentials._workload_identity_audience(full) == full + assert ( + google_credentials._workload_identity_audience( + "projects/123/providers/test" + ) + == "//iam.googleapis.com/projects/123/providers/test" ) - - assert calls == [("central-project", "wif-credentials")] - assert destination.logger == "policyengine-observability" + assert google_credentials._workload_identity_audience("custom") == "custom" diff --git a/tests/test_google_destination.py b/tests/test_google_destination.py deleted file mode 100644 index fffb9c0..0000000 --- a/tests/test_google_destination.py +++ /dev/null @@ -1,334 +0,0 @@ -from __future__ import annotations - -import json -import math - -import pytest - -from policyengine_observability.config import ObservabilityConfig -from policyengine_observability.destinations import ( - GoogleCloudLoggingDestination, - google_cloud_logging, - normalize_payload, -) -from policyengine_observability.destinations.base import ( - accepts_keyword, - clamped, -) - - -class Unprintable: - def __str__(self) -> str: - raise RuntimeError("cannot stringify") - - -class FakeLogger: - def __init__(self) -> None: - self.calls = [] - - def log_struct(self, payload, **kwargs) -> None: - self.calls.append((payload, kwargs)) - - -class FakeGapicApi: - def __init__(self) -> None: - self.calls = [] - - def write_log_entries(self, *args, **kwargs) -> None: - self.calls.append((args, kwargs)) - - -class FakeLoggingApi: - def __init__(self) -> None: - self._gapic_api = FakeGapicApi() - - -class FakeClient: - def __init__(self, *, gapic: bool = False) -> None: - self.project = "resolved-project" - self.fake_logger = FakeLogger() - self.log_names = [] - if gapic: - self.logging_api = FakeLoggingApi() - - def logger(self, log_name: str) -> FakeLogger: - self.log_names.append(log_name) - return self.fake_logger - - -def _google_destination(monkeypatch, client, **kwargs): - monkeypatch.setattr( - google_cloud_logging, - "load_google_credentials", - lambda *, prefer_workload_identity: None, - ) - monkeypatch.setattr( - google_cloud_logging, - "configure_google_application_credentials", - lambda: None, - ) - return GoogleCloudLoggingDestination( - project=None, - log_name="policyengine-observability", - client_factory=lambda _project, _credentials: client, - **kwargs, - ) - - -@pytest.mark.parametrize( - ("value", "expected"), - [ - (5.0, 5.0), - ("5", 5.0), - (0, 0.5), - (-3, 0.5), - (1000, 60.0), - (float("inf"), 10.0), - (float("nan"), 10.0), - (None, 10.0), - ("garbage", 10.0), - ], -) -def test_clamped_bounds_and_rejects_non_finite(value, expected) -> None: - result = clamped(value, low=0.5, high=60.0, default=10.0) - - assert result == expected - assert math.isfinite(result) - - -def test_accepts_keyword_covers_named_var_keyword_and_uninspectable() -> None: - def named(payload, *, timestamp=None): - pass - - def var_keyword(payload, **kwargs): - pass - - def blind(payload): - pass - - assert accepts_keyword(named, "timestamp") is True - assert accepts_keyword(var_keyword, "timestamp") is True - assert accepts_keyword(blind, "timestamp") is False - # Builtins without introspectable signatures degrade to False - # instead of raising at construction time. - assert accepts_keyword(min, "timestamp") is False - - -def test_normalize_payload_recursively_stringifies_unsafe_values() -> None: - normalized = normalize_payload( - { - "keep": "value", - "drop_none": None, - "bytes": b"value", - "list": [1, Unprintable()], - "nested": {"object": object()}, - } - ) - - assert normalized["keep"] == "value" - assert normalized["drop_none"] is None - assert normalized["bytes"] == "value" - assert normalized["list"] == [1, ""] - assert normalized["nested"]["object"].startswith(" None: - monkeypatch.setattr( - google_cloud_logging, - "load_google_credentials", - lambda *, prefer_workload_identity: None, - ) - monkeypatch.setattr( - google_cloud_logging, - "configure_google_application_credentials", - lambda: None, - ) - client = FakeClient() - destination = GoogleCloudLoggingDestination( - project=None, - log_name="policyengine-observability", - client_factory=lambda _project, _credentials: client, - ) - - destination.emit( - { - "schema_version": "policyengine.observability.request.v1", - "service_name": "svc", - "service_role": "api", - "environment": "production", - "request_id": "request-1", - "trace_id": "abc123", - "span_id": "def456", - "path": "/calculate", - "object": object(), - }, - log_type="request", - severity="ERROR", - ) - - payload, kwargs = client.fake_logger.calls[0] - assert client.log_names == ["policyengine-observability"] - assert payload["object"].startswith(" None: - client = FakeClient(gapic=True) - - destination = _google_destination( - monkeypatch, client, write_timeout_seconds=5.0 - ) - # The write path under log_struct funnels through this method; the - # rebinding must inject the bounded retry and per-call timeout. - client.logging_api._gapic_api.write_log_entries(request="sentinel") - - assert destination.write_timeout_seconds == 5.0 - ((args, kwargs),) = client.logging_api._gapic_api.calls - assert kwargs["request"] == "sentinel" - assert kwargs["timeout"] == 5.0 - assert kwargs["retry"].timeout == 5.0 - - -def test_google_destination_clamps_write_timeout(monkeypatch) -> None: - destination = _google_destination( - monkeypatch, FakeClient(gapic=True), write_timeout_seconds=0.0 - ) - - assert destination.write_timeout_seconds == 0.5 - - -def test_google_destination_without_gapic_transport_still_works( - monkeypatch, -) -> None: - client = FakeClient() - - destination = _google_destination(monkeypatch, client) - destination.emit({"event": "x"}, log_type="event", severity="INFO") - - assert len(client.fake_logger.calls) == 1 - - -def test_google_destination_forwards_enqueue_timestamp(monkeypatch) -> None: - from datetime import UTC, datetime - - client = FakeClient() - destination = _google_destination(monkeypatch, client) - stamp = datetime(2026, 7, 8, 12, 0, 0, tzinfo=UTC) - - destination.emit( - {"event": "x"}, log_type="event", severity="INFO", timestamp=stamp - ) - destination.emit({"event": "y"}, log_type="event", severity="INFO") - - (_, stamped_kwargs), (_, plain_kwargs) = client.fake_logger.calls - assert stamped_kwargs["timestamp"] is stamp - assert "timestamp" not in plain_kwargs - - -def test_google_destination_close_closes_client(monkeypatch) -> None: - class ClosableFakeClient(FakeClient): - def __init__(self) -> None: - super().__init__() - self.closed = 0 - - def close(self) -> None: - self.closed += 1 - - client = ClosableFakeClient() - destination = _google_destination(monkeypatch, client) - - destination.close() - - assert client.closed == 1 - - -def test_google_destination_close_tolerates_closeless_client( - monkeypatch, -) -> None: - destination = _google_destination(monkeypatch, FakeClient()) - - destination.close() # FakeClient has no close; must be a no-op - - -def test_google_destination_suppresses_instrumentation_entry( - monkeypatch, -) -> None: - logging_v2 = pytest.importorskip("google.cloud.logging_v2") - monkeypatch.setattr( - logging_v2, "_instrumentation_emitted", False, raising=False - ) - - _google_destination(monkeypatch, FakeClient()) - - assert logging_v2._instrumentation_emitted is True - - -def test_google_factory_reads_write_timeout_env(monkeypatch) -> None: - captured = {} - - class StubDestination: - def __init__(self, **kwargs) -> None: - captured.update(kwargs) - - monkeypatch.setattr( - google_cloud_logging, "GoogleCloudLoggingDestination", StubDestination - ) - monkeypatch.setenv("OBSERVABILITY_GOOGLE_WRITE_TIMEOUT_SECONDS", "2.5") - - from policyengine_observability.destinations.registry import ( - destination_strategy, - ) - - destination_strategy("google_cloud_logging").factory( - config=ObservabilityConfig(google_cloud_project="proj"), - loggers={}, - serializer=json.dumps, - ) - - assert captured["project"] == "proj" - assert captured["write_timeout_seconds"] == 2.5 - - -def test_google_factory_write_timeout_defaults_without_env( - monkeypatch, -) -> None: - captured = {} - - class StubDestination: - def __init__(self, **kwargs) -> None: - captured.update(kwargs) - - monkeypatch.setattr( - google_cloud_logging, "GoogleCloudLoggingDestination", StubDestination - ) - monkeypatch.delenv( - "OBSERVABILITY_GOOGLE_WRITE_TIMEOUT_SECONDS", raising=False - ) - - from policyengine_observability.destinations.registry import ( - destination_strategy, - ) - - destination_strategy("google_cloud_logging").factory( - config=ObservabilityConfig(google_cloud_project="proj"), - loggers={}, - serializer=json.dumps, - ) - - assert captured["write_timeout_seconds"] == 10.0 - - -# ── Stdout formatters ──────────────────────────────────────────────────── diff --git a/tests/test_log_profiles.py b/tests/test_log_profiles.py deleted file mode 100644 index ddade7a..0000000 --- a/tests/test_log_profiles.py +++ /dev/null @@ -1,292 +0,0 @@ -from __future__ import annotations - -import json - -import pytest -from fakes import RecordingLogger, make_manager - -from policyengine_observability.config import ObservabilityConfig -from policyengine_observability.destinations.queued import ( - QueuedLogDestination, -) - -PLATFORM_MARKERS = ( - "OBSERVABILITY_LOG_PROFILE", - "OBSERVABILITY_PLATFORM", - "K_SERVICE", - "MODAL_ENVIRONMENT", - "MODAL_TASK_ID", - "OBSERVABILITY_LOG_DESTINATIONS", - "OBSERVABILITY_STDOUT_FORMAT", - "OBSERVABILITY_GOOGLE_CLOUD_PROJECT", - "GOOGLE_CLOUD_PROJECT", - "GCP_PROJECT", - "GCLOUD_PROJECT", -) - - -@pytest.fixture(autouse=True) -def _clean_environment(monkeypatch): - for name in PLATFORM_MARKERS: - monkeypatch.delenv(name, raising=False) - - -def _from_env(monkeypatch, **env): - for name, value in env.items(): - monkeypatch.setenv(name, value) - return ObservabilityConfig.from_env(service_name="svc") - - -def test_explicit_gcp_agent_profile(monkeypatch) -> None: - config = _from_env(monkeypatch, OBSERVABILITY_LOG_PROFILE="gcp-agent") - - assert config.log_profile == "gcp-agent" - assert config.log_destinations == ("stdout",) - assert config.stdout_format == "google" - assert config.config_warnings == () - - -def test_profile_name_accepts_underscore_variant(monkeypatch) -> None: - config = _from_env(monkeypatch, OBSERVABILITY_LOG_PROFILE=" GCP_Agent ") - - assert config.log_profile == "gcp-agent" - assert config.config_warnings == () - - -def test_explicit_gcp_direct_profile_with_project(monkeypatch) -> None: - config = _from_env( - monkeypatch, - OBSERVABILITY_LOG_PROFILE="gcp-direct", - OBSERVABILITY_GOOGLE_CLOUD_PROJECT="proj", - ) - - assert config.log_profile == "gcp-direct" - assert config.log_destinations == ("stdout", "google_cloud_logging") - assert config.stdout_format == "plain" - assert config.config_warnings == () - - -def test_gcp_direct_without_project_downgrades_with_warning( - monkeypatch, -) -> None: - config = _from_env(monkeypatch, OBSERVABILITY_LOG_PROFILE="gcp-direct") - - assert config.log_profile == "plain-sync" - assert config.log_destinations == ("stdout",) - assert config.stdout_format == "plain" - assert len(config.config_warnings) == 1 - # The requirement comes from the strategy registration, and the - # warning names the actual profile — no hard-coded backend text. - assert "gcp-direct" in config.config_warnings[0] - assert "google_cloud_project" in config.config_warnings[0] - - -def test_explicit_plain_sync_profile_is_kill_switch(monkeypatch) -> None: - config = _from_env( - monkeypatch, - OBSERVABILITY_LOG_PROFILE="plain-sync", - OBSERVABILITY_PLATFORM="modal", - OBSERVABILITY_GOOGLE_CLOUD_PROJECT="proj", - ) - - assert config.log_profile == "plain-sync" - assert config.log_destinations == ("stdout",) - assert config.stdout_format == "plain" - - -def test_unknown_profile_falls_back_to_plain_sync_with_warning( - monkeypatch, -) -> None: - config = _from_env(monkeypatch, OBSERVABILITY_LOG_PROFILE="gcp-agnet") - - assert config.log_profile == "plain-sync" - assert config.log_destinations == ("stdout",) - assert any("gcp-agnet" in w for w in config.config_warnings) - - -def test_auto_detects_cloud_run_via_observability_platform( - monkeypatch, -) -> None: - config = _from_env(monkeypatch, OBSERVABILITY_PLATFORM="google_cloud_run") - - assert config.log_profile == "gcp-agent" - assert config.stdout_format == "google" - - -def test_auto_detects_modal_via_observability_platform(monkeypatch) -> None: - config = _from_env( - monkeypatch, - OBSERVABILITY_PLATFORM="modal", - OBSERVABILITY_GOOGLE_CLOUD_PROJECT="proj", - ) - - assert config.log_profile == "gcp-direct" - assert config.log_destinations == ("stdout", "google_cloud_logging") - - -def test_auto_detects_cloud_run_via_k_service(monkeypatch) -> None: - config = _from_env(monkeypatch, K_SERVICE="household-api") - - assert config.log_profile == "gcp-agent" - - -def test_auto_detects_modal_via_task_marker(monkeypatch) -> None: - config = _from_env( - monkeypatch, - MODAL_TASK_ID="ta-123", - OBSERVABILITY_GOOGLE_CLOUD_PROJECT="proj", - ) - - assert config.log_profile == "gcp-direct" - - -def test_auto_detects_modal_via_environment_marker(monkeypatch) -> None: - config = _from_env( - monkeypatch, - MODAL_ENVIRONMENT="main", - OBSERVABILITY_GOOGLE_CLOUD_PROJECT="proj", - ) - - assert config.log_profile == "gcp-direct" - - -def test_auto_detected_modal_without_project_downgrades(monkeypatch) -> None: - """The deployment-realistic failure: Modal markers present but no - resolvable project must land on plain-sync with a warning, exactly - like the explicit profile.""" - config = _from_env(monkeypatch, MODAL_TASK_ID="ta-123") - - assert config.log_profile == "plain-sync" - assert config.log_destinations == ("stdout",) - assert any("google_cloud_project" in w for w in config.config_warnings) - - -def test_observability_platform_beats_generic_markers(monkeypatch) -> None: - config = _from_env( - monkeypatch, - OBSERVABILITY_PLATFORM="google_cloud_run", - MODAL_TASK_ID="ta-123", - ) - - assert config.log_profile == "gcp-agent" - - -def test_auto_without_markers_preserves_caller_defaults(monkeypatch) -> None: - config = ObservabilityConfig.from_env( - service_name="svc", - default_log_destinations=("stdout", "custom"), - ) - - assert config.log_profile == "auto" - assert config.log_destinations == ("stdout", "custom") - assert config.stdout_format == "plain" - assert config.config_warnings == () - - -def test_explicit_destination_env_overrides_profile(monkeypatch) -> None: - config = _from_env( - monkeypatch, - OBSERVABILITY_LOG_PROFILE="gcp-agent", - OBSERVABILITY_LOG_DESTINATIONS="stdout,google_cloud_logging", - OBSERVABILITY_GOOGLE_CLOUD_PROJECT="proj", - ) - - assert config.log_destinations == ("stdout", "google_cloud_logging") - # The profile's other half still applies. - assert config.stdout_format == "google" - - -def test_explicit_stdout_format_env_overrides_profile(monkeypatch) -> None: - config = _from_env( - monkeypatch, - OBSERVABILITY_LOG_PROFILE="gcp-agent", - OBSERVABILITY_STDOUT_FORMAT="plain", - ) - - assert config.stdout_format == "plain" - assert config.log_destinations == ("stdout",) - - -def test_sync_profiles_build_no_queued_destinations(monkeypatch) -> None: - for profile in ("gcp-agent", "plain-sync"): - config = _from_env(monkeypatch, OBSERVABILITY_LOG_PROFILE=profile) - manager, failures = make_manager(config) - manager.configure() - - assert not any( - isinstance(destination, QueuedLogDestination) - for destination in manager.destinations - ) - assert failures == [] - - -def test_gcp_agent_profile_formats_stdout_through_manager( - monkeypatch, -) -> None: - """End-to-end wiring: the profile's formatter half must survive the - manager's build path, not just direct construction.""" - config = _from_env( - monkeypatch, - OBSERVABILITY_LOG_PROFILE="gcp-agent", - OBSERVABILITY_GOOGLE_CLOUD_PROJECT="proj", - ) - logger = RecordingLogger() - manager, failures = make_manager(config, loggers={"event": logger}) - manager.configure() - - manager.emit( - {"event": "x", "trace_id": "abc"}, log_type="event", severity="INFO" - ) - - line = json.loads(logger.lines[0][1]) - assert line["logging.googleapis.com/labels"]["log_type"] == "event" - assert line["logging.googleapis.com/trace"] == "projects/proj/traces/abc" - assert failures == [] - - -def test_manager_reports_profile_warnings_once(monkeypatch) -> None: - config = _from_env(monkeypatch, OBSERVABILITY_LOG_PROFILE="bogus") - manager, failures = make_manager(config) - manager.configure() - - warnings = [ - str(exc) - for operation, exc, _fields in failures - if operation == "logging.profile_config" - ] - assert len(warnings) == 1 - assert "bogus" in warnings[0] - manager.close() - - -def test_gcp_direct_profile_builds_queued_google(monkeypatch) -> None: - from policyengine_observability.destinations import ( - google_cloud_logging as google_module, - ) - - class StubGoogle: - name = "google_cloud_logging" - - def __init__(self, **kwargs) -> None: - self.kwargs = kwargs - - def emit(self, payload, *, log_type, severity, timestamp=None): - pass - - monkeypatch.setattr( - google_module, "GoogleCloudLoggingDestination", StubGoogle - ) - config = _from_env( - monkeypatch, - OBSERVABILITY_LOG_PROFILE="gcp-direct", - OBSERVABILITY_GOOGLE_CLOUD_PROJECT="proj", - ) - manager, failures = make_manager(config) - manager.configure() - - stdout_destination, queued = manager.destinations - assert isinstance(queued, QueuedLogDestination) - assert isinstance(queued.inner, StubGoogle) - assert queued.inner.kwargs["project"] == "proj" - assert failures == [] - manager.close() diff --git a/tests/test_otel.py b/tests/test_otel.py new file mode 100644 index 0000000..3c249b1 --- /dev/null +++ b/tests/test_otel.py @@ -0,0 +1,677 @@ +from __future__ import annotations + +import asyncio +import io +from datetime import UTC, datetime, timedelta + +from conftest import make_config, records +from opentelemetry.sdk.trace.export import ( + SimpleSpanProcessor, + SpanExportResult, +) +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + +from policyengine_observability import ( + GoogleIdTokenAuth, + OTelConfig, + OTLPExporterConfig, + configure, +) +from policyengine_observability.diagnostics import Diagnostics +from policyengine_observability.google_auth import ( + _google_grpc_credentials, + _google_http_session, + _google_id_token_credentials, + _workload_identity_audience, +) +from policyengine_observability.otel import ( + OTelRuntime, + SpanHandle, + _build_metric_exporter, + _build_span_exporter, + _exporter_kwargs, + _http_signal_endpoint, + _lazy_metric_exporter_class, + _LazySpanExporter, + captured_at_is_recent, +) + + +def _runtime_with_spans(): + config = make_config(otel=OTelConfig(enabled=True)) + runtime = configure(config) + runtime._delivery._stdout = io.StringIO() + exporter = InMemorySpanExporter() + runtime._otel._tracer_provider.add_span_processor( + SimpleSpanProcessor(exporter) + ) + return runtime, exporter + + +def test_owned_provider_creates_local_spans_metrics_and_resources() -> None: + runtime, exporter = _runtime_with_spans() + resource = runtime._otel.resource_attributes() + assert resource["service.name"] == "test-api" + assert resource["deployment.environment.name"] == "test" + assert runtime._otel.tracer is not None + assert runtime._otel.meter is not None + with runtime.operation("simulation.run"): + with runtime.span("simulation.calculate"): + runtime.event("calculation.started") + runtime._otel.force_flush(1) + spans = exporter.get_finished_spans() + assert {span.name for span in spans} == { + "simulation.run", + "simulation.calculate", + } + child = next(span for span in spans if span.name == "simulation.calculate") + parent = next(span for span in spans if span.name == "simulation.run") + assert child.parent.span_id == parent.context.span_id + runtime.shutdown() + + +def test_sensitive_values_are_redacted_from_spans_and_logs() -> None: + config = make_config( + otel=OTelConfig(enabled=True), + application_attribute_keys=frozenset({"backend"}), + sensitive_values=("secret-value",), + ) + runtime = configure(config) + runtime._delivery._stdout = io.StringIO() + exporter = InMemorySpanExporter() + runtime._otel._tracer_provider.add_span_processor( + SimpleSpanProcessor(exporter) + ) + + try: + with runtime.operation( + "simulation.run", + attributes={"backend": "prefix-secret-value-suffix"}, + ): + raise ValueError("secret-value operation failed") + except ValueError: + pass + + span = exporter.get_finished_spans()[0] + assert span.attributes["backend"] == "prefix-[REDACTED]-suffix" + event = span.events[0] + assert event.name == "exception" + assert event.attributes["exception.message"] == ( + "[REDACTED] operation failed" + ) + assert "secret-value" not in event.attributes["exception.stacktrace"] + assert "[REDACTED]" in event.attributes["exception.stacktrace"] + item = records(runtime._delivery._stdout)[0] + assert item["attributes"]["backend"] == "prefix-[REDACTED]-suffix" + assert item["error.message"] == "[REDACTED] operation failed" + runtime.shutdown() + + +def test_incoming_trace_context_correlates_log_and_response() -> None: + runtime, exporter = _runtime_with_spans() + trace_id = "1" * 32 + parent_id = "2" * 16 + runtime.begin_request( + headers={"traceparent": f"00-{trace_id}-{parent_id}-01"}, + method="GET", + route="/health", + ) + headers = runtime.response_headers() + assert headers["traceparent"].startswith(f"00-{trace_id}-") + runtime.end_request(status_code=200) + item = records(runtime._delivery._stdout)[0] + assert item["trace_id"] == trace_id + assert "logging.googleapis.com/trace" not in item + span = exporter.get_finished_spans()[0] + assert span.parent.span_id == int(parent_id, 16) + runtime.shutdown() + + +def test_malformed_trace_context_starts_new_trace() -> None: + runtime, exporter = _runtime_with_spans() + runtime.begin_request( + headers={"traceparent": "malformed"}, + method="GET", + route="/health", + ) + runtime.end_request(status_code=200) + span = exporter.get_finished_spans()[0] + assert span.context.trace_id != 0 + assert span.parent is None + runtime.shutdown() + + +def test_server_error_response_sets_span_error_status() -> None: + from opentelemetry.trace import StatusCode + + runtime, exporter = _runtime_with_spans() + runtime.begin_request( + headers={}, + method="GET", + route="/unavailable", + ) + + runtime.end_request(status_code=503) + + span = exporter.get_finished_spans()[0] + assert span.status.status_code is StatusCode.ERROR + assert span.events == () + runtime.shutdown() + + +def test_recent_async_context_is_parent_and_old_context_is_link() -> None: + runtime, exporter = _runtime_with_spans() + trace_id = "3" * 32 + parent_id = "4" * 16 + recent = { + "traceparent": f"00-{trace_id}-{parent_id}-01", + "request_id": "dispatch-request", + "captured_at": datetime.now(UTC).isoformat(), + } + with runtime.operation("recent", remote_context=recent): + runtime.event("recent.event") + + old = { + **recent, + "captured_at": (datetime.now(UTC) - timedelta(minutes=10)).isoformat(), + } + with runtime.operation("old", remote_context=old): + pass + spans = {span.name: span for span in exporter.get_finished_spans()} + assert spans["recent"].parent.span_id == int(parent_id, 16) + assert spans["old"].parent is None + assert spans["old"].links[0].context.span_id == int(parent_id, 16) + runtime.shutdown() + + +def test_retry_forces_span_link_even_when_context_is_recent() -> None: + runtime, exporter = _runtime_with_spans() + remote = { + "traceparent": f"00-{'5' * 32}-{'6' * 16}-01", + "captured_at": datetime.now(UTC).isoformat(), + } + with runtime.operation( + "retry", remote_context=remote, independent_retry=True + ): + pass + span = exporter.get_finished_spans()[0] + assert span.parent is None + assert len(span.links) == 1 + runtime.shutdown() + + +def test_external_provider_mode_uses_caller_provider(monkeypatch) -> None: + class Provider: + def get_tracer(self, *_args): + return object() + + class MeterProvider: + def get_meter(self, *_args): + return None + + tracer_provider = Provider() + meter_provider = MeterProvider() + monkeypatch.setattr( + "opentelemetry.trace.get_tracer_provider", lambda: tracer_provider + ) + monkeypatch.setattr( + "opentelemetry.metrics.get_meter_provider", lambda: meter_provider + ) + otel = OTelRuntime( + make_config(otel=OTelConfig(enabled=True, provider_mode="external")), + Diagnostics(), + ) + assert otel._tracer_provider is tracer_provider + assert otel._meter_provider is meter_provider + assert not otel._owns_tracer_provider + + +def test_lazy_span_exporter_contains_failure() -> None: + diagnostics = Diagnostics() + exporter = _LazySpanExporter( + lambda: (_ for _ in ()).throw(ConnectionError("dns failure")), + diagnostics, + ) + assert exporter.export([]) is SpanExportResult.FAILURE + assert diagnostics.count("spans.export_failure") == 1 + assert exporter.force_flush() + exporter.shutdown() + + +def test_lazy_metric_exporter_contains_failure() -> None: + from opentelemetry.sdk.metrics.export import MetricExportResult + + diagnostics = Diagnostics() + exporter = _lazy_metric_exporter_class()( + lambda: (_ for _ in ()).throw(PermissionError("denied")), + diagnostics, + ) + assert exporter.export(object()) is MetricExportResult.FAILURE + assert diagnostics.count("metrics.export_failure") == 1 + assert exporter.force_flush() + exporter.shutdown() + + +def test_async_context_age_and_http_endpoint_helpers() -> None: + assert captured_at_is_recent(datetime.now(UTC).isoformat(), 300) + assert not captured_at_is_recent("invalid", 300) + assert not captured_at_is_recent(None, 300) + assert not captured_at_is_recent( + (datetime.now(UTC) + timedelta(seconds=10)).isoformat(), 300 + ) + assert _http_signal_endpoint("https://collector", "traces") == ( + "https://collector/v1/traces" + ) + assert ( + _http_signal_endpoint("https://collector/v1/traces", "traces") + == "https://collector/v1/traces" + ) + + +def test_otlp_exporter_builders_apply_protocol_endpoint_and_timeout( + monkeypatch, +) -> None: + from opentelemetry.exporter.otlp.proto.grpc import ( + metric_exporter as grpc_metric, + ) + from opentelemetry.exporter.otlp.proto.grpc import ( + trace_exporter as grpc_trace, + ) + from opentelemetry.exporter.otlp.proto.http import ( + metric_exporter as http_metric, + ) + from opentelemetry.exporter.otlp.proto.http import ( + trace_exporter as http_trace, + ) + + created: list[tuple[str, dict]] = [] + + def constructor(name): + return lambda **kwargs: created.append((name, kwargs)) or name + + monkeypatch.setattr( + grpc_trace, "OTLPSpanExporter", constructor("grpc-span") + ) + monkeypatch.setattr( + grpc_metric, "OTLPMetricExporter", constructor("grpc-metric") + ) + monkeypatch.setattr( + http_trace, "OTLPSpanExporter", constructor("http-span") + ) + monkeypatch.setattr( + http_metric, "OTLPMetricExporter", constructor("http-metric") + ) + grpc = OTLPExporterConfig( + endpoint="collector:4317", + protocol="grpc", + headers=(("x-test", "value"),), + timeout_seconds=2, + ) + assert _build_span_exporter(grpc) == "grpc-span" + assert _build_metric_exporter(grpc) == "grpc-metric" + http = OTLPExporterConfig( + endpoint="https://collector/", + protocol="http/protobuf", + timeout_seconds=3, + ) + assert _build_span_exporter(http) == "http-span" + assert _build_metric_exporter(http) == "http-metric" + assert created[2][1]["endpoint"] == "https://collector/v1/traces" + assert created[3][1]["endpoint"] == "https://collector/v1/metrics" + assert created[0][1]["headers"] == {"x-test": "value"} + assert created[0][1]["timeout"] == 2 + + exact = OTLPExporterConfig( + endpoint="https://collector/custom-signal", + protocol="http/protobuf", + endpoint_mode="signal", + ) + assert _build_span_exporter(exact) == "http-span" + assert _build_metric_exporter(exact) == "http-metric" + assert created[4][1]["endpoint"] == "https://collector/custom-signal" + assert created[5][1]["endpoint"] == "https://collector/custom-signal" + + +def test_google_exporter_kwargs_select_protocol_credentials( + monkeypatch, +) -> None: + monkeypatch.setattr( + "policyengine_observability.google_auth._google_grpc_credentials", + lambda audience: f"grpc:{audience}", + ) + monkeypatch.setattr( + "policyengine_observability.google_auth._google_http_session", + lambda audience, headers: (audience, dict(headers)), + ) + grpc = _exporter_kwargs( + OTLPExporterConfig( + endpoint="collector:4317", + auth=GoogleIdTokenAuth("https://collector"), + ) + ) + assert grpc["credentials"] == "grpc:https://collector" + http = _exporter_kwargs( + OTLPExporterConfig( + endpoint="https://collector", + protocol="http/protobuf", + auth=GoogleIdTokenAuth("https://collector"), + headers=(("x", "y"),), + ) + ) + assert http["session"] == ("https://collector", {"x": "y"}) + assert "headers" not in http + + +def test_google_id_token_uses_modal_workload_identity(monkeypatch) -> None: + from google.auth import identity_pool, impersonated_credentials + + calls: dict[str, object] = {} + + class Source: + def __init__(self, **kwargs): + calls["source"] = kwargs + + class Target: + def __init__(self, **kwargs): + calls["target"] = kwargs + + class Identity: + def __init__(self, **kwargs): + calls["identity"] = kwargs + + monkeypatch.setattr(identity_pool, "Credentials", Source) + monkeypatch.setattr(impersonated_credentials, "Credentials", Target) + monkeypatch.setattr( + impersonated_credentials, "IDTokenCredentials", Identity + ) + monkeypatch.setenv("MODAL_IDENTITY_TOKEN", "modal-token") + monkeypatch.setenv( + "OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER", + "projects/123/providers/example", + ) + monkeypatch.setenv( + "OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL", + "modal@central.iam.gserviceaccount.com", + ) + result = _google_id_token_credentials("https://collector") + assert isinstance(result, Identity) + assert "credential_source" not in calls["source"] + supplier = calls["source"]["subject_token_supplier"] + assert supplier.get_subject_token(None, None) == "modal-token" + assert calls["identity"]["target_audience"] == "https://collector" + + +def test_google_id_token_falls_back_to_application_credentials( + monkeypatch, +) -> None: + from google.oauth2 import id_token + + monkeypatch.delenv("MODAL_IDENTITY_TOKEN", raising=False) + monkeypatch.delenv("OBSERVABILITY_GOOGLE_OIDC_TOKEN", raising=False) + monkeypatch.setattr( + id_token, + "fetch_id_token_credentials", + lambda audience, request: (audience, request), + ) + result = _google_id_token_credentials("https://collector") + assert result[0] == "https://collector" + + +def test_google_transport_helpers(monkeypatch) -> None: + import grpc + from google.auth.transport import grpc as google_grpc + from google.auth.transport import requests as google_requests + + monkeypatch.setattr( + "policyengine_observability.google_auth._google_id_token_credentials", + lambda audience: f"token:{audience}", + ) + monkeypatch.setattr( + google_grpc, + "AuthMetadataPlugin", + lambda credentials, request, default_host: ( + credentials, + default_host, + ), + ) + monkeypatch.setattr(grpc, "ssl_channel_credentials", lambda: "ssl") + monkeypatch.setattr( + grpc, "metadata_call_credentials", lambda plugin: ("metadata", plugin) + ) + monkeypatch.setattr( + grpc, + "composite_channel_credentials", + lambda ssl, metadata: (ssl, metadata), + ) + assert _google_grpc_credentials("https://collector.example") == ( + "ssl", + ( + "metadata", + ("token:https://collector.example", "collector.example"), + ), + ) + + class Session: + def __init__(self, credentials): + self.credentials = credentials + self.headers = {} + + monkeypatch.setattr(google_requests, "AuthorizedSession", Session) + session = _google_http_session("https://collector", {"x": "y"}) + assert session.credentials == "token:https://collector" + assert session.headers == {"x": "y"} + + assert _workload_identity_audience("projects/123/provider") == ( + "//iam.googleapis.com/projects/123/provider" + ) + + +def test_otel_runtime_contains_span_and_propagation_failures( + monkeypatch, +) -> None: + from opentelemetry import trace + from opentelemetry.trace.propagation.tracecontext import ( + TraceContextTextMapPropagator, + ) + + diagnostics = Diagnostics() + otel = OTelRuntime( + make_config(otel=OTelConfig(enabled=False)), diagnostics + ) + + class BrokenTracer: + def start_as_current_span(self, *_args, **_kwargs): + raise RuntimeError("start failed") + + otel.tracer = BrokenTracer() + assert otel.start_span("broken") is None + + monkeypatch.setattr( + trace, + "get_current_span", + lambda *_args: (_ for _ in ()).throw(RuntimeError("context failed")), + ) + assert otel.current_correlation() == {} + otel.set_span_attributes({"key": "value"}) + assert otel.remote_span_context({}) is None + + monkeypatch.setattr( + TraceContextTextMapPropagator, + "extract", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + ValueError("malformed") + ), + ) + monkeypatch.setattr( + TraceContextTextMapPropagator, + "inject", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + ValueError("unavailable") + ), + ) + assert otel.extract({"traceparent": "bad"}) is None + otel.inject({}) + assert diagnostics.count("failure.otel.span_start") == 1 + assert diagnostics.count("failure.otel.context_extract") == 1 + assert diagnostics.count("failure.otel.context_inject") == 1 + + +def test_otel_end_span_metrics_flush_and_shutdown_fail_open() -> None: + diagnostics = Diagnostics() + otel = OTelRuntime( + make_config(otel=OTelConfig(enabled=False)), diagnostics + ) + + class Span: + def __init__(self): + self.events = [] + self.status = None + + def add_event(self, name, attributes): + self.events.append((name, attributes)) + + def set_status(self, status): + self.status = status + + class Manager: + def __init__(self): + self.args = None + + def __exit__(self, *args): + self.args = args + + span = Span() + manager = Manager() + error = ValueError("application") + otel.end_span(SpanHandle(manager=manager, span=span), error) + assert span.events[0][0] == "exception" + assert span.events[0][1]["exception.message"] == "application" + assert manager.args[1] is error + otel.end_span(None) + + class Instrument: + def add(self, *_args, **_kwargs): + raise RuntimeError("metric failed") + + otel._error_count = Instrument() + otel.record_error({"outcome": "error"}) + otel.record_dropped("log") + + class Provider: + def force_flush(self, **_kwargs): + raise TimeoutError("flush") + + def shutdown(self, **_kwargs): + raise TimeoutError("shutdown") + + provider = Provider() + otel._tracer_provider = provider + otel._meter_provider = provider + otel._owns_tracer_provider = True + otel._owns_meter_provider = True + otel.force_flush(0.01) + otel.shutdown(0.01) + assert diagnostics.count("failure.otel.metric_record") == 1 + assert diagnostics.count("failure.otel.force_flush") == 2 + assert diagnostics.count("failure.otel.trace_shutdown") == 1 + assert diagnostics.count("failure.otel.metric_shutdown") == 1 + + +def test_lazy_exporters_delegate_flush_and_shutdown_failures() -> None: + diagnostics = Diagnostics() + + class SpanDelegate: + def export(self, spans): + return SpanExportResult.SUCCESS + + def force_flush(self, _timeout): + raise TimeoutError("flush") + + def shutdown(self): + raise RuntimeError("shutdown") + + span_exporter = _LazySpanExporter(lambda: SpanDelegate(), diagnostics) + assert span_exporter.export([]) is SpanExportResult.SUCCESS + assert not span_exporter.force_flush() + span_exporter.shutdown() + + class MetricDelegate: + def export(self, *_args, **_kwargs): + from opentelemetry.sdk.metrics.export import MetricExportResult + + return MetricExportResult.SUCCESS + + def force_flush(self, _timeout): + raise TimeoutError("flush") + + def shutdown(self, **_kwargs): + raise RuntimeError("shutdown") + + metric_exporter = _lazy_metric_exporter_class()( + lambda: MetricDelegate(), diagnostics + ) + from opentelemetry.sdk.metrics.export import MetricExportResult + + assert metric_exporter.export(object()) is MetricExportResult.SUCCESS + assert not metric_exporter.force_flush() + metric_exporter.shutdown() + assert diagnostics.count("failure.otel.span_flush") == 1 + assert diagnostics.count("failure.otel.span_exporter_shutdown") == 1 + assert diagnostics.count("failure.otel.metric_flush") == 1 + assert diagnostics.count("failure.otel.metric_exporter_shutdown") == 1 + + +def test_queue_depth_observation_is_bounded_and_failure_is_local() -> None: + diagnostics = Diagnostics() + otel = OTelRuntime( + make_config(otel=OTelConfig(enabled=False)), + diagnostics, + queue_depth=lambda: -4, + ) + observation = otel._observe_queue_depth(None)[0] + assert observation.value == 0 + otel._queue_depth_callback = lambda: (_ for _ in ()).throw( + RuntimeError("queue unavailable") + ) + assert otel._observe_queue_depth(None) == [] + assert diagnostics.count("failure.otel.queue_depth") == 1 + + +def test_concurrent_requests_keep_trace_and_span_context_separate() -> None: + runtime, exporter = _runtime_with_spans() + + async def request(request_id: str) -> None: + runtime.begin_request( + headers={"X-PolicyEngine-Request-Id": request_id}, + method="GET", + route="/concurrent", + ) + async with runtime.span(f"child.{request_id}"): + await asyncio.sleep(0) + runtime.event("inside") + runtime.end_request(status_code=200) + + async def run() -> None: + await asyncio.gather(request("one"), request("two")) + + asyncio.run(run()) + completion_records = [ + item + for item in records(runtime._delivery._stdout) + if item.get("event.name") == "request.completed" + ] + traces = { + item["request.id"]: item["trace_id"] for item in completion_records + } + assert traces["one"] != traces["two"] + child_spans = { + span.name: span + for span in exporter.get_finished_spans() + if span.parent + } + for request_id in ("one", "two"): + child = child_spans[f"child.{request_id}"] + assert format(child.context.trace_id, "032x") == traces[request_id] + runtime.shutdown() diff --git a/tests/test_otel_destinations.py b/tests/test_otel_destinations.py new file mode 100644 index 0000000..08e29f3 --- /dev/null +++ b/tests/test_otel_destinations.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from policyengine_observability import ( + DeploymentIdentity, + GoogleIdTokenAuth, + ObservabilityConfig, + OTLPExporterConfig, + ServiceIdentity, +) + + +def test_signal_specific_otlp_environment_configuration(monkeypatch) -> None: + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://common") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "https://traces") + monkeypatch.setenv( + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "https://metrics" + ) + monkeypatch.setenv( + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", "trace-key=trace-value" + ) + monkeypatch.setenv( + "OTEL_EXPORTER_OTLP_METRICS_HEADERS", "metric-key=metric-value" + ) + + config = ObservabilityConfig.from_env( + service=ServiceIdentity("service", "namespace", "1", "api"), + deployment=DeploymentIdentity("test", "other"), + ) + + assert config.otel.traces == OTLPExporterConfig( + endpoint="https://traces", + endpoint_mode="signal", + headers=(("trace-key", "trace-value"),), + ) + assert config.otel.metrics == OTLPExporterConfig( + endpoint="https://metrics", + endpoint_mode="signal", + headers=(("metric-key", "metric-value"),), + ) + + +def test_google_authentication_is_an_explicit_exporter_strategy( + monkeypatch, +) -> None: + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "collector:4317") + monkeypatch.setenv( + "POLICYENGINE_OTEL_GOOGLE_AUDIENCE", "https://collector" + ) + + config = ObservabilityConfig.from_env( + service=ServiceIdentity("service", "namespace", "1", "worker"), + deployment=DeploymentIdentity("test", "modal"), + ) + + assert isinstance(config.otel.traces.auth, GoogleIdTokenAuth) + assert isinstance(config.otel.metrics.auth, GoogleIdTokenAuth) + assert config.otel.traces.auth.audience == "https://collector" diff --git a/tests/test_public_api.py b/tests/test_public_api.py deleted file mode 100644 index a24f474..0000000 --- a/tests/test_public_api.py +++ /dev/null @@ -1,126 +0,0 @@ -from __future__ import annotations - -import asyncio -from enum import StrEnum - -import policyengine_observability as observability -from policyengine_observability import ( - ObservabilityConfig, - ObservabilityRuntime, -) - - -class SegmentName(StrEnum): - LOAD = "load" - - -class RecordingInstrument: - def __init__(self) -> None: - self.calls = [] - - def add(self, value, attributes=None) -> None: - self.calls.append(("add", value, attributes)) - - def record(self, value, attributes=None) -> None: - self.calls.append(("record", value, attributes)) - - -def test_public_wrappers_delegate_to_configured_runtime() -> None: - runtime = ObservabilityRuntime( - ObservabilityConfig(service_name="svc"), - segment_registry=SegmentName, - ) - runtime.operation_duration = RecordingInstrument() - runtime.segment_duration = RecordingInstrument() - runtime.operations = RecordingInstrument() - runtime.errors = RecordingInstrument() - runtime.failover_events = RecordingInstrument() - observability.set_observability_runtime(runtime) - - with observability.operation("job", flavor="cli"): - observability.set_attribute("tool", "worker") - observability.record_event("fallback_selected", reason="forced") - observability.record_error( - RuntimeError("handled"), - handled=True, - include_stack=False, - ) - with observability.segment(SegmentName.LOAD, tool="loader"): - observability.mark("custom_ms", 12.3) - observability.mark_ttft() - observability.mark_ttft_attribute() - handle = observability.start_scope({}, name="nested") - observability.annotate(handle, model="claude") - observability.end_scope(handle) - - assert observability.current_context() is None - assert observability.current_operation() is None - assert observability.traceparent_header() is None - assert observability.capture_context() is None - assert runtime.operation_duration.calls - assert runtime.segment_duration.calls - assert runtime.errors.calls - assert runtime.failover_events.calls - - -def test_public_decorators_support_sync_and_async_functions() -> None: - runtime = ObservabilityRuntime(ObservabilityConfig(service_name="svc")) - runtime.operation_duration = RecordingInstrument() - runtime.segment_duration = RecordingInstrument() - runtime.operations = RecordingInstrument() - observability.set_observability_runtime(runtime) - - @observability.entrypoint("sync_job", flavor="cli") - def sync_job() -> str: - return "sync" - - @observability.segment("async_step", flavor="worker") - async def async_step() -> str: - return "async" - - assert sync_job() == "sync" - assert asyncio.run(async_step()) == "async" - observability.instrument_fastapi(object()) - observability.instrument_httpx() - observability.shutdown_tracing() - observability.shutdown_observability() - observability.restart_observability() - - assert runtime.operation_duration.calls - assert runtime.segment_duration.calls - assert runtime.log_destination_manager.configured is True - - -def test_setting_public_runtime_clears_stale_context() -> None: - old_runtime = ObservabilityRuntime(ObservabilityConfig(service_name="old")) - observability.set_observability_runtime(old_runtime) - handle = old_runtime.start_operation("old_job") - - try: - new_runtime = ObservabilityRuntime( - ObservabilityConfig(service_name="new") - ) - observability.set_observability_runtime(new_runtime) - - assert observability.current_operation() is None - assert observability.current_context() is None - finally: - old_runtime.end_operation(handle) - - -def test_public_async_segment_and_collect_timings_wrappers() -> None: - runtime = ObservabilityRuntime(ObservabilityConfig(service_name="svc")) - observability.set_observability_runtime(runtime) - - async def run() -> dict[str, float]: - with observability.collect_timings("job") as timings: - async with observability.asegment("load"): - pass - return timings - - assert "load_ms" in asyncio.run(run()) - - -def test_registration_hooks_are_exported_at_top_level() -> None: - assert callable(observability.register_destination) - assert callable(observability.register_stdout_formatter) diff --git a/tests/test_queued_destination.py b/tests/test_queued_destination.py deleted file mode 100644 index b5b5364..0000000 --- a/tests/test_queued_destination.py +++ /dev/null @@ -1,472 +0,0 @@ -"""QueuedLogDestination tests. - -Determinism strategy: real listener threads are used, but sequencing is -via threading.Event only — close()/stop() drain to a sentinel enqueued -behind the records, so "close then assert delivered" is deterministic. -The only real-time waits are bounded joins that are themselves the -behavior under test, with generous ceilings. -""" - -from __future__ import annotations - -import time -from datetime import UTC, datetime - -import pytest -from fakes import ( - BlockingDestination, - FailingDestination, - RecordingDestination, - TimestampBlindDestination, - make_manager, -) - -from policyengine_observability.config import ObservabilityConfig -from policyengine_observability.destinations.queued import ( - QueuedLogDestination, -) -from policyengine_observability.destinations.registry import ( - destination_strategy, -) - - -def _queued(inner, **kwargs): - failures = [] - destination = QueuedLogDestination( - inner=inner, - on_failure=lambda operation, exc, **fields: failures.append( - (operation, fields) - ), - **kwargs, - ) - return destination, failures - - -def _raise_on_failure(*args, **kwargs): - raise RuntimeError("reporting channel is broken") - - -def _release_abandoned_listener(destination, blocking_inner) -> None: - """Let an abandoned listener thread exit so tests do not leak it.""" - blocking_inner.release.set() - try: - destination._queue.put_nowait(None) # the stdlib sentinel - except Exception: - pass - - -def test_close_drains_all_records_in_order_with_enqueue_timestamps() -> None: - inner = RecordingDestination() - destination, failures = _queued(inner) - before = datetime.now(UTC) - - for index in range(5): - destination.emit({"index": index}, log_type="event", severity="INFO") - destination.close(5.0) - - assert [payload["index"] for payload, *_ in inner.calls] == [ - 0, - 1, - 2, - 3, - 4, - ] - for _payload, log_type, severity, timestamp in inner.calls: - assert log_type == "event" - assert severity == "INFO" - assert timestamp.tzinfo is not None - assert before <= timestamp <= datetime.now(UTC) - assert failures == [] - - -def test_timestamp_not_forwarded_to_blind_inner() -> None: - inner = TimestampBlindDestination() - destination, failures = _queued(inner) - - destination.emit({"event": "x"}, log_type="event", severity="INFO") - destination.close(5.0) - - assert inner.calls == [({"event": "x"}, "event", "INFO")] - assert failures == [] - - -def test_timestamp_forwarded_to_var_keyword_inner() -> None: - class KwargsDestination: - name = "kwargs" - - def __init__(self) -> None: - self.kwargs = [] - - def emit(self, payload, **kwargs) -> None: - self.kwargs.append(kwargs) - - inner = KwargsDestination() - destination, failures = _queued(inner) - - destination.emit({"event": "x"}, log_type="event", severity="INFO") - destination.close(5.0) - - assert inner.kwargs[0]["timestamp"].tzinfo is not None - assert failures == [] - - -def test_uninspectable_inner_emit_degrades_to_timestamp_blind() -> None: - class BuiltinEmitDestination: - # inspect.signature(min) raises; construction must survive and - # fall back to timestamp-blind delivery. - name = "builtin-emit" - emit = min - - destination, _failures = _queued(BuiltinEmitDestination()) - - handler = destination._listener.handlers[0] - assert handler.forward_timestamp is False - destination.close(5.0) - - -def test_payload_snapshot_taken_at_enqueue() -> None: - inner = RecordingDestination() - destination, _failures = _queued(inner) - payload = {"nested": {"value": 1}} - - destination.emit(payload, log_type="event", severity="INFO") - payload["nested"]["value"] = 2 - destination.close(5.0) - - ((delivered, *_),) = inner.calls - assert delivered["nested"]["value"] == 1 - - -def test_overflow_drops_newest_and_throttles_reports() -> None: - inner = BlockingDestination() - destination, failures = _queued(inner, maxsize=10, drop_report_interval=3) - - destination.emit({"index": 0}, log_type="event", severity="INFO") - assert inner.entered.wait(5) - # The listener holds record 0; fill the 10-slot queue, then drop 4. - started = time.monotonic() - for index in range(1, 15): - destination.emit({"index": index}, log_type="event", severity="INFO") - assert time.monotonic() - started < 1.0 - - drops = [fields for op, fields in failures if op == "logging.queue_drop"] - assert [d["dropped_total"] for d in drops] == [1, 3] - assert all(d["reason"] == "full" for d in drops) - assert all(d["queue_maxsize"] == 10 for d in drops) - assert destination._drops.count == 4 - - inner.release.set() - destination.close(5.0) - - -def test_close_with_stuck_write_is_bounded_and_terminal() -> None: - inner = BlockingDestination() - destination, failures = _queued(inner) - destination.emit({"index": 0}, log_type="event", severity="INFO") - assert inner.entered.wait(5) - destination.emit({"index": 1}, log_type="event", severity="INFO") - - started = time.monotonic() - destination.close(0.05) - elapsed = time.monotonic() - started - - assert elapsed < 1.0 - timeouts = [ - fields - for op, fields in failures - if op == "logging.queue_close_timeout" - ] - assert len(timeouts) == 1 - assert timeouts[0]["pending_records"] >= 1 - - destination.emit({"index": 2}, log_type="event", severity="INFO") - drops = [fields for op, fields in failures if op == "logging.queue_drop"] - assert drops[-1]["reason"] == "closed" - - _release_abandoned_listener(destination, inner) - - -def test_timed_out_close_leaves_inner_unclosed() -> None: - """An abandoned listener may be mid-write; the inner destination - must not be closed underneath it.""" - inner = BlockingDestination() - destination, _failures = _queued(inner) - destination.emit({"index": 0}, log_type="event", severity="INFO") - assert inner.entered.wait(5) - - destination.close(0.05) - - assert inner.closed == 0 - _release_abandoned_listener(destination, inner) - - -def test_close_with_sentinel_blocked_by_full_queue_is_bounded() -> None: - inner = BlockingDestination() - destination, failures = _queued(inner, maxsize=10) - destination.emit({"index": 0}, log_type="event", severity="INFO") - assert inner.entered.wait(5) - for index in range(1, 11): - destination.emit({"index": index}, log_type="event", severity="INFO") - - started = time.monotonic() - destination.close(0.05) - elapsed = time.monotonic() - started - - assert elapsed < 1.0 - assert any(op == "logging.queue_close_timeout" for op, _fields in failures) - - _release_abandoned_listener(destination, inner) - - -def test_listener_survives_write_failures_and_throttles_reports() -> None: - inner = FailingDestination(fail_first=3) - destination, failures = _queued(inner) - - for index in range(5): - destination.emit({"index": index}, log_type="event", severity="INFO") - destination.close(5.0) - - assert [p["index"] for p in inner.delivered] == [3, 4] - writes = [fields for op, fields in failures if op == "logging.queue_write"] - # Throttle: failure 1 reports, failures 2-3 are under the interval. - assert [w["write_failures_total"] for w in writes] == [1] - assert writes[0]["destination"] == "failing" - - -def test_listener_survives_broken_reporting_channel() -> None: - """The write-failure report itself is guarded on the listener - thread: a raising on_failure must not kill delivery.""" - inner = FailingDestination(fail_first=1) - destination = QueuedLogDestination( - inner=inner, on_failure=_raise_on_failure - ) - - destination.emit({"index": 0}, log_type="event", severity="INFO") - destination.emit({"index": 1}, log_type="event", severity="INFO") - destination.close(5.0) - - assert [p["index"] for p in inner.delivered] == [1] - - -def test_emit_never_raises() -> None: - inner = RecordingDestination() - destination = QueuedLogDestination( - inner=inner, on_failure=_raise_on_failure - ) - - poisoned = {} - poisoned["self"] = poisoned # RecursionError inside normalize - destination.emit(poisoned, log_type="event", severity="INFO") - - destination.close(5.0) - destination.emit({"event": "x"}, log_type="event", severity="INFO") - - blocked = BlockingDestination() - full_destination = QueuedLogDestination( - inner=blocked, on_failure=_raise_on_failure, maxsize=10 - ) - for index in range(12): - full_destination.emit( - {"index": index}, log_type="event", severity="INFO" - ) - - blocked.release.set() - full_destination.close(5.0) - - -def test_atexit_registered_on_construction_unregistered_on_close( - monkeypatch, -) -> None: - registered = [] - unregistered = [] - monkeypatch.setattr( - "policyengine_observability.destinations.queued.atexit.register", - lambda func: registered.append(func), - ) - monkeypatch.setattr( - "policyengine_observability.destinations.queued.atexit.unregister", - lambda func: unregistered.append(func), - ) - inner = RecordingDestination() - - destination, failures = _queued(inner) - destination.close(5.0) - destination.close(5.0) # idempotent: no second stop, no reports - - assert registered == [destination.close] - assert unregistered == [destination.close] - assert failures == [] - - -def test_close_forwards_to_inner_close_only_when_drained() -> None: - class ClosableInner(RecordingDestination): - def __init__(self) -> None: - super().__init__() - self.closed = 0 - - def close(self) -> None: - self.closed += 1 - - inner = ClosableInner() - destination, _failures = _queued(inner) - destination.emit({"event": "x"}, log_type="event", severity="INFO") - destination.close(5.0) - - assert inner.closed == 1 - - -def test_inner_close_failure_is_reported_not_raised() -> None: - class ExplodingClose(RecordingDestination): - def close(self) -> None: - raise RuntimeError("client teardown failed") - - destination, failures = _queued(ExplodingClose()) - - destination.close(5.0) # must not raise (atexit calls this) - - assert any(op == "logging.destination_close" for op, _ in failures) - - -def test_knobs_are_clamped() -> None: - inner = RecordingDestination() - destination, _failures = _queued( - inner, maxsize=float("inf"), close_timeout_seconds=-5 - ) - - assert destination.maxsize == 1000 - assert destination.close_timeout_seconds == 0.0 - destination.close(5.0) - - -@pytest.mark.parametrize( - ("kwargs", "attribute", "expected"), - [ - ({"maxsize": 3}, "maxsize", 10), - ({"maxsize": 10**6}, "maxsize", 100_000), - ({"close_timeout_seconds": 100}, "close_timeout_seconds", 30.0), - ], -) -def test_knob_boundaries_are_pinned(kwargs, attribute, expected) -> None: - """Pin the clamp bounds so a low/high swap cannot pass silently.""" - destination, _failures = _queued(RecordingDestination(), **kwargs) - - assert getattr(destination, attribute) == expected - destination.close(5.0) - - -# ── Destination registry ───────────────────────────────────────────────── - - -def test_registry_builds_remote_wrapped_and_inline_bare( - fake_remote_strategy, -) -> None: - manager, _failures = make_manager( - ObservabilityConfig(log_destinations=(fake_remote_strategy, "stdout")) - ) - manager.configure() - - remote, inline = manager.destinations - assert isinstance(remote, QueuedLogDestination) - assert remote.name == "queued_recording" - assert not isinstance(inline, QueuedLogDestination) - manager.close() - - -def test_registry_unknown_name_reports_config_failure() -> None: - manager, failures = make_manager( - ObservabilityConfig(log_destinations=("nonexistent",)) - ) - manager.configure() - - assert any( - op == "logging.destination_config" - and fields.get("destination") == "nonexistent" - for op, _exc, fields in failures - ) - - -def test_google_strategy_registered_as_remote_with_aliases() -> None: - canonical = destination_strategy("google_cloud_logging") - assert canonical is not None - assert canonical.transport == "remote" - assert canonical.required_config == ("google_cloud_project",) - assert destination_strategy("google") is canonical - assert destination_strategy(" Google-Cloud ") is canonical - - -# ── Manager integration ────────────────────────────────────────────────── - - -def test_manager_breaker_never_disables_queued_destination() -> None: - inner = FailingDestination() - manager, failures = make_manager() - destination = QueuedLogDestination( - inner=inner, on_failure=manager.on_failure - ) - manager.destinations = [destination] - manager.configured = True - - for _ in range(5): - manager.emit({"event": "x"}, log_type="event", severity="INFO") - destination.close(5.0) - - operations = [op for op, *_ in failures] - assert destination in manager.destinations - assert "logging.destination_disabled" not in operations - assert "logging.destination_emit" not in operations - - -def test_manager_passes_queue_knobs_to_queued_destination( - fake_remote_strategy, -) -> None: - manager, _failures = make_manager( - ObservabilityConfig( - log_destinations=(fake_remote_strategy,), - log_queue_maxsize=50, - log_queue_close_timeout_seconds=7.5, - ) - ) - manager.configure() - - queued = manager.destinations[0] - assert queued.maxsize == 50 - assert queued.close_timeout_seconds == 7.5 - manager.close() - - -def test_reconfigure_closes_previous_destinations( - fake_remote_strategy, -) -> None: - manager, _failures = make_manager( - ObservabilityConfig(log_destinations=(fake_remote_strategy,)) - ) - manager.configure() - first = manager.destinations[0] - manager._consecutive_failures[id(first)] = 2 - - manager.configure() - - assert first._closed is True - assert manager._consecutive_failures == {} - assert manager.destinations[0] is not first - manager.close() - - -def test_reconfigure_after_close_resumes_delivery( - fake_remote_strategy, -) -> None: - """The fork/snapshot story: close → drops → configure() → a fresh - listener delivers again.""" - manager, _failures = make_manager( - ObservabilityConfig(log_destinations=(fake_remote_strategy,)) - ) - manager.configure() - manager.close() - manager.emit({"event": "dropped"}, log_type="event", severity="INFO") - - manager.configure() - manager.emit({"event": "delivered"}, log_type="event", severity="INFO") - manager.close(5.0) - - rebuilt = manager.destinations[0] - assert [p["event"] for p in rebuilt.inner.payloads] == ["delivered"] diff --git a/tests/test_release_scripts.py b/tests/test_release_scripts.py deleted file mode 100644 index 1ae568c..0000000 --- a/tests/test_release_scripts.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import annotations - -import importlib.util -from pathlib import Path - -import pytest - -REPO_ROOT = Path(__file__).resolve().parents[1] - - -def _load_script(path: str, module_name: str): - spec = importlib.util.spec_from_file_location( - module_name, - REPO_ROOT / path, - ) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - return module - - -bump_version = _load_script(".github/bump_version.py", "bump_version") -fetch_version = _load_script(".github/fetch_version.py", "fetch_version") - - -def test_infer_bump_uses_towncrier_fragment_precedence(tmp_path) -> None: - changelog_dir = tmp_path / "changelog.d" - changelog_dir.mkdir() - (changelog_dir / "feature.added.md").write_text("Add a feature.\n") - (changelog_dir / "api.breaking.md").write_text("Break an API.\n") - - assert bump_version.infer_bump(changelog_dir) == "major" - - -def test_infer_bump_rejects_missing_fragments(tmp_path) -> None: - changelog_dir = tmp_path / "changelog.d" - changelog_dir.mkdir() - - with pytest.raises(SystemExit): - bump_version.infer_bump(changelog_dir) - - -def test_current_version_prefers_highest_pyproject_changelog_or_tag( - tmp_path, - monkeypatch, -) -> None: - pyproject = tmp_path / "pyproject.toml" - pyproject.write_text('[project]\nversion = "0.1.0"\n') - changelog = tmp_path / "CHANGELOG.md" - changelog.write_text("## [0.2.0] - 2026-06-22\n") - monkeypatch.setattr( - bump_version, - "get_git_tag_versions", - lambda _repo_root: ["0.3.0"], - ) - - assert ( - bump_version.get_current_version(pyproject, changelog, tmp_path) - == "0.3.0" - ) - - -def test_update_file_replaces_project_version(tmp_path) -> None: - pyproject = tmp_path / "pyproject.toml" - pyproject.write_text('[project]\nversion = "0.1.0"\n') - - bump_version.update_file(pyproject, "0.2.0") - - assert 'version = "0.2.0"' in pyproject.read_text() - - -def test_fetch_version_reads_project_version(tmp_path) -> None: - pyproject = tmp_path / "pyproject.toml" - pyproject.write_text('[project]\nversion = "1.2.3"\n') - - assert fetch_version.fetch_version(pyproject) == "1.2.3" diff --git a/tests/test_runtime.py b/tests/test_runtime.py new file mode 100644 index 0000000..49db755 --- /dev/null +++ b/tests/test_runtime.py @@ -0,0 +1,457 @@ +from __future__ import annotations + +import asyncio +import logging +import os +import select +import signal +import threading +import time + +import pytest +from conftest import make_config, make_runtime, records + +from policyengine_observability import ( + REQUEST_ID_HEADER, + ConfigurationError, + LoggingConfig, + OTelConfig, + configure, + instrument_logging, +) + + +def test_operation_context_emits_one_completion(runtime) -> None: + observed, output = runtime + with observed.operation("simulation.run", attributes={"backend": "modal"}): + observed.event("simulation.started") + + emitted = records(output) + assert [item["event.name"] for item in emitted] == [ + "simulation.started", + "operation.completed", + ] + completion = emitted[-1] + assert completion["operation.name"] == "simulation.run" + assert completion["outcome"] == "success" + assert completion["attributes"] == {"backend": "modal"} + + +def test_span_does_not_emit_operation_completion(runtime) -> None: + observed, output = runtime + with observed.span("simulation.build"): + observed.event("inside") + assert [item["event.name"] for item in records(output)] == ["inside"] + + +def test_sync_decorator_preserves_result(runtime) -> None: + observed, output = runtime + + @observed.operation("calculate") + def calculate(value: int) -> int: + return value * 2 + + assert calculate(4) == 8 + assert records(output)[0]["event.name"] == "operation.completed" + + +def test_async_context_and_decorator_preserve_result(runtime) -> None: + observed, output = runtime + + @observed.span("child") + async def child() -> str: + await asyncio.sleep(0) + return "done" + + async def run() -> str: + async with observed.operation("async.run"): + return await child() + + assert asyncio.run(run()) == "done" + assert [item["event.name"] for item in records(output)] == [ + "operation.completed" + ] + + +def test_application_exception_is_same_object_when_export_fails( + runtime, +) -> None: + observed, _output = runtime + application_error = RuntimeError("application failure") + + def broken_emit(_record) -> None: + raise OSError("logging unavailable") + + observed._delivery.emit = broken_emit + + @observed.operation("broken") + def fail() -> None: + raise application_error + + with pytest.raises(RuntimeError) as caught: + fail() + assert caught.value is application_error + assert observed.diagnostics.count("failure.record.emit") == 1 + + +def test_observability_entry_failure_does_not_change_return( + monkeypatch, +) -> None: + observed, _output = make_runtime() + + def fail_start(*_args, **_kwargs): + raise RuntimeError("instrumentation failed") + + monkeypatch.setattr(observed, "_start_operation", fail_start) + + @observed.operation("work") + def work() -> int: + return 42 + + assert work() == 42 + assert observed.diagnostics.count("failure.operation.start") == 1 + observed.shutdown() + + +def test_process_control_exception_from_instrumentation_is_not_caught( + monkeypatch, +) -> None: + observed, _output = make_runtime() + + def stop(*_args, **_kwargs): + raise KeyboardInterrupt + + monkeypatch.setattr(observed, "_start_operation", stop) + with pytest.raises(KeyboardInterrupt): + with observed.operation("work"): + pass + observed.shutdown() + + +def test_request_accepts_valid_id_and_emits_once(runtime) -> None: + observed, output = runtime + request_id = observed.begin_request( + headers={REQUEST_ID_HEADER.lower(): "request-123"}, + method="get", + route="/household/", + ) + assert request_id == "request-123" + assert observed.response_headers()[REQUEST_ID_HEADER] == "request-123" + observed.end_request(status_code=200) + observed.end_request(status_code=200) + completion = records(output)[0] + assert completion["event.name"] == "request.completed" + assert completion["request.id"] == "request-123" + assert completion["http.request.method"] == "GET" + assert completion["http.route"] == "/household/" + assert completion["http.response.status_code"] == 200 + + +def test_request_replaces_malformed_id_and_classifies_status(runtime) -> None: + observed, output = runtime + request_id = observed.begin_request( + headers={REQUEST_ID_HEADER: "invalid id with spaces"}, + method="POST", + route="", + ) + assert request_id != "invalid id with spaces" + observed.update_request_route("/economy") + observed.end_request(status_code=404) + assert records(output)[0]["outcome"] == "client_error" + + +def test_request_exception_is_error(runtime) -> None: + observed, output = runtime + observed.begin_request(headers={}, method="GET", route="/fail") + error = ValueError("bad") + observed.record_exception(error, handled=False, status_code=500) + observed.end_request(status_code=500, error=error) + emitted = records(output) + assert emitted[0]["event.name"] == "exception.recorded" + assert emitted[1]["event.name"] == "request.completed" + assert emitted[1]["outcome"] == "error" + + +def test_set_context_and_capture_context_are_allowlisted(runtime) -> None: + observed, _output = runtime + observed.begin_request( + headers={REQUEST_ID_HEADER: "request-1"}, + method="POST", + route="/simulation", + ) + observed.set_context( + auth_result="accepted", + job_id="job-1", + household="must-not-appear", + unapproved="must-not-appear", + ) + captured = observed.capture_context() + assert captured["request_id"] == "request-1" + assert captured["job_id"] == "job-1" + assert "auth_result" not in captured + assert "household" not in captured + assert "unapproved" not in captured + assert "captured_at" in captured + assert observed.diagnostics.count("attributes.omitted") == 2 + observed.end_request(status_code=202) + + +def test_two_runtimes_keep_identity_and_context_separate() -> None: + first, first_output = make_runtime() + second_config = make_config( + service=make_config().service.__class__( + "second-api", "policyengine.test", "1", "worker" + ) + ) + second = configure(second_config) + import io + + second_output = io.StringIO() + second._delivery._stdout = second_output + first.begin_request( + headers={REQUEST_ID_HEADER: "first-request"}, + method="GET", + route="/first", + ) + second.event("second.event") + first.end_request(status_code=200) + assert records(first_output)[0]["request.id"] == "first-request" + second_record = records(second_output)[0] + assert second_record["service.name"] == "second-api" + assert "request.id" not in second_record + first.shutdown() + second.shutdown() + + +def test_concurrent_async_tasks_do_not_share_request_ids() -> None: + observed, output = make_runtime() + + async def request(request_id: str) -> None: + observed.begin_request( + headers={REQUEST_ID_HEADER: request_id}, + method="GET", + route="/task", + ) + await asyncio.sleep(0) + observed.event("task.event") + observed.end_request(status_code=200) + + async def run() -> None: + await asyncio.gather(request("one"), request("two")) + + asyncio.run(run()) + grouped = { + item["request.id"] for item in records(output) if "request.id" in item + } + assert grouped == {"one", "two"} + observed.shutdown() + + +def test_standard_logging_is_explicit_and_idempotent(runtime) -> None: + observed, output = runtime + logger = logging.getLogger("tests.application") + logger.handlers.clear() + logger.propagate = False + first = instrument_logging(logger, observed) + second = instrument_logging(logger, observed) + assert first is second + logger.warning( + "structured warning", + extra={"policyengine_attributes": {"backend": "local"}}, + ) + item = records(output)[0] + assert item["message"] == "structured warning" + assert item["severity"] == "WARNING" + assert item["attributes"] == {"backend": "local"} + + +def test_shutdown_is_bounded_repeatable_and_restartable(runtime) -> None: + observed, _output = runtime + observed.shutdown() + observed.shutdown() + assert observed._closed + observed.restart_after_snapshot() + assert not observed._closed + + +def test_shutdown_contains_delivery_failure_and_continues_cleanup( + monkeypatch, +) -> None: + observed, _output = make_runtime( + logging=LoggingConfig(shutdown_timeout_seconds=0.125), + otel=OTelConfig(enabled=False, shutdown_timeout_seconds=0.375), + ) + otel_timeouts: list[float] = [] + + class OTel: + def shutdown(self, timeout: float) -> None: + otel_timeouts.append(timeout) + + def fail_close(timeout: float) -> None: + assert timeout == 0.125 + raise RuntimeError("logging close failed") + + logger = logging.getLogger("tests.shutdown.cleanup") + logger.handlers.clear() + logger.propagate = False + handler = instrument_logging(logger, observed) + observed._otel = OTel() + monkeypatch.setattr(observed._delivery, "close", fail_close) + + observed.shutdown() + observed.shutdown() + + assert otel_timeouts == [0.375] + assert handler not in logger.handlers + assert observed.diagnostics.count("failure.logging.shutdown") == 1 + + +def test_shutdown_bounds_otel_with_its_own_timeout() -> None: + observed, _output = make_runtime( + logging=LoggingConfig(shutdown_timeout_seconds=0), + otel=OTelConfig(enabled=False, shutdown_timeout_seconds=0.01), + ) + started = threading.Event() + release = threading.Event() + + class BlockingOTel: + def shutdown(self, timeout: float) -> None: + assert timeout == 0.01 + started.set() + release.wait(1) + + observed._otel = BlockingOTel() + before = time.perf_counter() + observed.shutdown() + elapsed = time.perf_counter() - before + + assert started.is_set() + assert elapsed < 0.2 + assert observed.diagnostics.count("shutdown.timeout") == 1 + release.set() + + +def test_shutdown_contains_otel_coordinator_failure(monkeypatch) -> None: + observed, _output = make_runtime() + + def fail_bounded_shutdown(*_args, **_kwargs) -> None: + raise RuntimeError("could not start shutdown worker") + + monkeypatch.setattr( + "policyengine_observability.runtime._run_bounded", + fail_bounded_shutdown, + ) + + observed.shutdown() + + assert observed.diagnostics.count("failure.otel.shutdown") == 1 + + +@pytest.mark.parametrize( + ("logging_timeout", "otel_timeout"), + [ + ("invalid", float("inf")), + (-1.0, 100.0), + ], +) +def test_invalid_shutdown_timeout_configuration_is_rejected( + logging_timeout, + otel_timeout, +) -> None: + with pytest.raises(ConfigurationError) as raised: + make_runtime( + logging=LoggingConfig(shutdown_timeout_seconds=logging_timeout), + otel=OTelConfig( + enabled=False, + shutdown_timeout_seconds=otel_timeout, + ), + ) + + message = str(raised.value) + assert "logging.shutdown_timeout_seconds" in message + assert "otel.shutdown_timeout_seconds" in message + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires POSIX fork") +def test_restart_after_snapshot_replaces_inherited_locked_state() -> None: + observed, _output = make_runtime() + observed._shutdown_lock.acquire() + observed.diagnostics._lock.acquire() + read_fd, write_fd = os.pipe() + child_pid = os.fork() + + if child_pid == 0: + os.close(read_fd) + try: + observed.restart_after_snapshot() + observed.diagnostics.increment("post_fork") + observed.shutdown() + os.write(write_fd, b"ok") + except BaseException as error: + os.write(write_fd, repr(error).encode()[:1_024]) + finally: + os.close(write_fd) + os._exit(0) + + os.close(write_fd) + result = b"" + try: + readable, _, _ = select.select([read_fd], [], [], 2) + if not readable: + os.kill(child_pid, signal.SIGKILL) + pytest.fail("child blocked on inherited observability state") + result = os.read(read_fd, 1_024) + finally: + observed.diagnostics._lock.release() + observed._shutdown_lock.release() + os.close(read_fd) + os.waitpid(child_pid, 0) + observed.shutdown() + + assert result == b"ok" + + +def test_restart_after_snapshot_rebuilds_process_local_components() -> None: + observed, _output = make_runtime() + observed.diagnostics.increment("before_snapshot") + observed.begin_request( + headers={REQUEST_ID_HEADER: "snapshotted-request"}, + method="GET", + route="/snapshot", + ) + inherited_delivery = observed._delivery + inherited_otel = observed._otel + + observed.restart_after_snapshot() + + assert observed._delivery is not inherited_delivery + assert observed._otel is not inherited_otel + assert observed.response_headers() == {} + assert observed.diagnostics.count("before_snapshot") == 0 + assert not observed._closed + observed.shutdown() + + +def test_local_drop_and_export_diagnostics_increment_otel_metrics() -> None: + observed, _output = make_runtime() + calls: list[tuple[str, str, int]] = [] + + class Metrics: + def record_dropped(self, name: str, value: int) -> None: + calls.append(("dropped", name, value)) + + def record_exporter_failure(self, name: str, value: int) -> None: + calls.append(("failure", name, value)) + + def shutdown(self, _timeout: float) -> None: + pass + + observed._otel = Metrics() + observed.diagnostics.increment("logs.dropped.queue_full", 2) + observed.diagnostics.increment("logs.export_failure", 1) + observed.diagnostics.increment("unrelated", 1) + assert calls == [ + ("dropped", "logs.dropped.queue_full", 2), + ("failure", "logs.export_failure", 1), + ] + observed.shutdown() diff --git a/tests/test_runtime_emission.py b/tests/test_runtime_emission.py deleted file mode 100644 index 8a57f91..0000000 --- a/tests/test_runtime_emission.py +++ /dev/null @@ -1,425 +0,0 @@ -from __future__ import annotations - -from types import SimpleNamespace - -from runtime_helpers import ( - FailingInstrument, - FailingLogDestination, - RecordingInstrument, - RecordingLogDestination, - SegmentName, - runtime, -) - -from policyengine_observability import ( - ObservabilityConfig, - ObservabilityRuntime, - RequestObservabilityContext, - coerce_segment_name, -) -from policyengine_observability.destinations import ( - google_cloud_logging as google_cloud_logging_module, -) - - -def test_trace_helpers_log_failures_without_throwing() -> None: - observed = runtime() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - observed.propagate = SimpleNamespace( - inject=lambda _carrier: (_ for _ in ()).throw(RuntimeError("inject")), - extract=lambda _carrier: (_ for _ in ()).throw( - RuntimeError("extract") - ), - ) - observed.trace = SimpleNamespace( - get_current_span=lambda: (_ for _ in ()).throw(RuntimeError("span")) - ) - - assert observed.traceparent_header() is None - assert observed._extract_context({"traceparent": "parent"}) is None - assert observed._current_span() is None - assert failures == [ - "request.traceparent_header", - "otel.extract_context", - "otel.current_span", - ] - - -def test_record_event_covers_operation_context_and_no_context_metrics() -> ( - None -): - observed = runtime() - observed.failover_events = RecordingInstrument() - - with observed.operation("worker", flavor="queue"): - observed.record_event("modal_retry", attempt=1, ignored=None) - - observed.record_event("fallback_without_context") - - assert len(observed.failover_events.calls) == 2 - assert observed.failover_events.calls[0][2]["operation"] == "worker" - assert observed.failover_events.calls[1][2]["event"] == ( - "fallback_without_context" - ) - - -def test_record_event_request_context_and_emit_log_skip_paths() -> None: - observed = runtime() - observed.failover_events = RecordingInstrument() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/event", - path="/event", - endpoint="event", - query_keys=[], - content_length_bytes=None, - inbound={}, - internal_dispatch=True, - ) - observed.begin_request(context) - observed.record_event("fallback_request", detail="request") - observed.emit_request_log(context) - observed.emit_request_log(context) - observed.teardown_request(None) - - operation = observed.start_operation("job")["operation"] - observed.emit_operation_log(operation) - observed.emit_operation_log(operation) - observed.end_operation({"operation": operation}) - - assert observed.failover_events.calls[0][2]["route"] == "/event" - assert context.emitted is True - assert operation.emitted is True - - -def test_operation_log_emits_to_configured_destinations_once() -> None: - observed = runtime() - destination = RecordingLogDestination() - observed.log_destination_manager.destinations = [destination] - observed.log_destination_manager.configured = True - - handle = observed.start_operation("job") - operation = handle["operation"] - observed.end_operation(handle) - observed.emit_operation_log(operation) - - assert len(destination.calls) == 1 - payload, log_type, severity = destination.calls[0] - assert payload["operation"] == "job" - assert payload["severity"] == "INFO" - assert log_type == "operation" - assert severity == "INFO" - - -def test_request_log_emits_to_configured_destination_once() -> None: - observed = runtime() - destination = RecordingLogDestination() - observed.log_destination_manager.destinations = [destination] - observed.log_destination_manager.configured = True - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/calculate", - path="/calculate", - endpoint="calculate", - query_keys=[], - content_length_bytes=None, - inbound={"client_ip": "203.0.113.1"}, - status_code=200, - ) - - observed.emit_request_log(context) - observed.emit_request_log(context) - - assert len(destination.calls) == 1 - payload, log_type, severity = destination.calls[0] - assert payload["request_id"] == "request-1" - assert payload["client_ip"] == "203.0.113.1" - assert payload["severity"] == "INFO" - assert log_type == "request" - assert severity == "INFO" - - -def test_request_log_reserved_fields_override_inbound_and_attributes() -> None: - observed = runtime() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/calculate", - path="/calculate", - endpoint="calculate", - query_keys=[], - content_length_bytes=None, - inbound={ - "request_id": "inbound-request", - "status_code": "inbound-status", - "duration_ms": "inbound-duration", - "segment_tree": "inbound-tree", - }, - attributes={ - "request_id": "attribute-request", - "status_code": "attribute-status", - "duration_ms": "attribute-duration", - "timings_ms": "attribute-timings", - "timing_counts": "attribute-counts", - "segment_tree": "attribute-tree", - "error": "attribute-error", - }, - status_code=204, - ) - - payload = context.as_log_record(trace_id=None, span_id=None) - assert payload["request_id"] == "request-1" - assert payload["status_code"] == 204 - assert isinstance(payload["duration_ms"], float) - assert isinstance(payload["timings_ms"], dict) - assert isinstance(payload["timing_counts"], dict) - assert payload["segment_tree"] == [] - assert payload["error"] is None - - -def test_event_log_emits_to_configured_destination() -> None: - observed = runtime() - destination = RecordingLogDestination() - observed.log_destination_manager.destinations = [destination] - observed.log_destination_manager.configured = True - - observed.record_event("custom_event", detail="value") - - assert len(destination.calls) == 1 - payload, log_type, severity = destination.calls[0] - assert payload["event"] == "custom_event" - assert payload["detail"] == "value" - assert payload["service_name"] == "svc" - assert payload["severity"] == "INFO" - assert log_type == "event" - assert severity == "INFO" - - -def test_log_severity_maps_status_codes_and_errors() -> None: - observed = runtime() - - assert observed._severity_for_log_record({"status_code": 200}) == "INFO" - assert observed._severity_for_log_record({"status_code": 404}) == ( - "WARNING" - ) - assert observed._severity_for_log_record({"status_code": "500"}) == ( - "ERROR" - ) - assert ( - observed._severity_for_log_record({"error": {"handled": False}}) - == "ERROR" - ) - assert observed._severity_for_log_record({"error": {"handled": True}}) == ( - "WARNING" - ) - - -def test_destination_failure_logs_internal_error_without_throwing() -> None: - observed = runtime() - recording = RecordingLogDestination() - observed.log_destination_manager.destinations = [ - FailingLogDestination(), - recording, - ] - observed.log_destination_manager.configured = True - - with observed.operation("job"): - pass - - assert any( - payload["event"] == "observability_internal_error" - and payload["operation"] == "logging.destination_emit" - for payload, _log_type, _severity in recording.calls - ) - assert any( - payload.get("operation") == "job" - for payload, _log_type, _severity in recording.calls - ) - - -def test_all_destination_failures_fall_back_to_stderr(capsys) -> None: - observed = runtime() - observed.log_destination_manager.destinations = [FailingLogDestination()] - observed.log_destination_manager.configured = True - - with observed.operation("job"): - pass - - stderr = capsys.readouterr().err - assert "observability_internal_error" in stderr - assert "logging.destination_emit" in stderr - - -def test_unknown_destination_falls_back_to_stdout() -> None: - observed = ObservabilityRuntime( - ObservabilityConfig( - service_name="svc", - otel_enabled=False, - log_destinations=("missing",), - ) - ) - - observed.configure() - - assert [ - destination.name - for destination in observed.log_destination_manager.destinations - ] == ["stdout"] - - -def test_google_destination_init_failure_falls_back_to_stdout( - monkeypatch, -) -> None: - def fail_google_destination(**_kwargs): - raise ImportError("google-cloud-logging missing") - - monkeypatch.setattr( - google_cloud_logging_module, - "GoogleCloudLoggingDestination", - fail_google_destination, - ) - observed = ObservabilityRuntime( - ObservabilityConfig( - service_name="svc", - otel_enabled=False, - log_destinations=("google_cloud_logging",), - ) - ) - - observed.configure() - - assert [ - destination.name - for destination in observed.log_destination_manager.destinations - ] == ["stdout"] - - -def test_disabled_configure_does_not_initialize_log_destinations( - monkeypatch, -) -> None: - def fail_google_destination(**_kwargs): - raise AssertionError("google destination should not initialize") - - monkeypatch.setattr( - google_cloud_logging_module, - "GoogleCloudLoggingDestination", - fail_google_destination, - ) - observed = ObservabilityRuntime( - ObservabilityConfig( - service_name="svc", - enabled=False, - log_destinations=("google_cloud_logging",), - ) - ) - - observed.configure() - - assert observed.log_destination_manager.destinations == [] - assert observed.log_destination_manager.configured is False - - -def test_record_segment_metric_covers_calculation_and_backend() -> None: - observed = runtime() - observed.segment_duration = RecordingInstrument() - observed.calculate_duration = RecordingInstrument() - observed.backend_duration = RecordingInstrument() - - observed.record_segment_metric( - "calculation", - 0.1, - {"backend": "modal"}, - backend_segment=True, - ) - - assert observed.segment_duration.calls - assert observed.calculate_duration.calls - assert observed.backend_duration.calls - - -def test_metric_recording_failures_are_logged() -> None: - observed = runtime() - observed.operation_duration = FailingInstrument() - observed.http_duration = FailingInstrument() - observed.segment_duration = FailingInstrument() - observed.errors = FailingInstrument() - observed.rate_limited = FailingInstrument() - observed.failover_events = FailingInstrument() - observed.active_requests = FailingInstrument() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - - observed.record_operation_metric(0.1, {}) - observed.record_request_metric(0.1, {}) - observed.record_segment_metric("load", 0.1, {}) - observed.record_error_metric({}) - observed.record_rate_limited_metric({}) - observed.record_failover_event_metric({}) - observed.record_active_request(1, {}) - - assert failures == [ - "metrics.record_operation", - "metrics.record_request", - "metrics.record_segment", - "metrics.record_error", - "metrics.record_rate_limited", - "metrics.record_failover_event", - "metrics.add_active_request", - ] - - -def test_private_safety_helpers_cover_fallback_paths( - monkeypatch, - capsys, -) -> None: - observed = runtime() - - class Unprintable: - def __str__(self) -> str: - raise RuntimeError("cannot stringify") - - assert observed._safe_str(Unprintable()) == "" - assert observed._safe_traceback(Unprintable()) == "" - - original_dumps = __import__("json").dumps - - def failing_dumps(payload, *args, **kwargs): - if payload.get("event") == "bad": - raise RuntimeError("json failed") - return original_dumps(payload, *args, **kwargs) - - monkeypatch.setattr("json.dumps", failing_dumps) - assert "observability_internal_error" in observed._json({"event": "bad"}) - - monkeypatch.setattr( - "policyengine_observability.runtime.INTERNAL_LOGGER.error", - lambda _message: (_ for _ in ()).throw(RuntimeError("logger failed")), - ) - observed.log_observability_failure("test", RuntimeError("failed")) - assert "observability_internal_error" in capsys.readouterr().err - - -def test_coerce_segment_name_validates_registry() -> None: - assert coerce_segment_name(SegmentName.LOAD, registry=SegmentName) == ( - "load", - True, - ) - assert coerce_segment_name("other", registry=SegmentName) == ( - "other", - False, - ) - assert coerce_segment_name("load", registry=["load"]) == ("load", True) - assert coerce_segment_name(SegmentName.LOAD, registry=None) == ( - "load", - True, - ) diff --git a/tests/test_runtime_operations.py b/tests/test_runtime_operations.py deleted file mode 100644 index 13f8e4a..0000000 --- a/tests/test_runtime_operations.py +++ /dev/null @@ -1,295 +0,0 @@ -from __future__ import annotations - -import asyncio -from types import SimpleNamespace - -import pytest -from runtime_helpers import ( - RecordingInstrument, - RecordingSpan, - RecordingTracer, - SegmentName, - runtime, -) - -from policyengine_observability import ( - RequestObservabilityContext, -) -from policyengine_observability import _state as state_module - - -def test_operation_context_manager_async_and_exception_paths() -> None: - async def run() -> None: - observed = runtime() - observed.errors = RecordingInstrument() - - with pytest.raises(RuntimeError, match="async failed"): - async with observed.operation("async_job", flavor="worker"): - raise RuntimeError("async failed") - - assert observed.errors.calls[0][2]["error_type"] == "RuntimeError" - assert observed.current_operation() is None - - asyncio.run(run()) - - -def test_start_operation_with_parent_context_attaches_and_detaches() -> None: - observed = runtime() - observed.tracer = RecordingTracer() - parent_context = object() - - handle = observed.start_operation( - "parented", - parent_context=parent_context, - ) - observed.end_operation(handle) - - assert observed.tracer.calls[0][0] == "parented" - assert observed.current_operation() is None - - -def test_operation_attach_detach_and_reset_failures_are_logged( - monkeypatch, -) -> None: - from opentelemetry import context as otel_context - - observed = runtime() - observed.tracer = RecordingTracer() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append((operation, fields.get("token"))) - ) - monkeypatch.setattr( - otel_context, - "attach", - lambda _context: (_ for _ in ()).throw(RuntimeError("attach failed")), - ) - - handle = observed.start_operation("job", parent_context=object()) - observed.end_operation(handle) - - monkeypatch.setattr( - otel_context, - "detach", - lambda _token: (_ for _ in ()).throw(RuntimeError("detach failed")), - ) - observed.end_operation( - { - "operation": None, - "context_token": object(), - "timings_token": object(), - "start_token": object(), - "operation_token": object(), - } - ) - - assert ("operation.context_attach", None) in failures - assert ("operation.context_detach", None) in failures - assert ("operation.context_reset", "timings_token") in failures - assert ("operation.context_reset", "start_token") in failures - assert ("operation.context_reset", "operation_token") in failures - - -def test_operation_end_and_start_failures_are_logged(monkeypatch) -> None: - class BrokenVar: - def set(self, _value): - raise RuntimeError("set failed") - - observed = runtime() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - operation_handle = observed.start_operation("job") - operation = operation_handle["operation"] - operation.metric_recorded = True - observed.complete_operation(operation) - observed.complete_operation = lambda _operation: (_ for _ in ()).throw( - RuntimeError("complete failed") - ) - observed.end_operation(operation_handle) - - monkeypatch.setattr(state_module, "_OPERATION_CONTEXT", BrokenVar()) - handle = observed.start_operation("job") - - assert handle["operation"] is None - assert failures == ["operation.end", "operation.start"] - - -def test_standalone_segment_creates_implicit_operation_metrics() -> None: - observed = runtime() - observed.segment_duration = RecordingInstrument() - observed.operation_duration = RecordingInstrument() - observed.operations = RecordingInstrument() - emitted_payloads = [] - observed.emit_operation_log = lambda operation: emitted_payloads.append( - operation.as_log_record(trace_id=None, span_id=None) - ) - - with observed.segment(SegmentName.LOAD, flavor="cli", tool="loader"): - pass - - _, _, segment_attributes = observed.segment_duration.calls[0] - _, _, operation_attributes = observed.operation_duration.calls[0] - assert segment_attributes["operation"] == "load" - assert segment_attributes["flavor"] == "cli" - assert segment_attributes["tool"] == "loader" - assert operation_attributes["operation"] == "load" - assert operation_attributes["flavor"] == "cli" - assert emitted_payloads[0]["segment_tree"][0]["name"] == "load" - assert emitted_payloads[0]["segment_tree"][0]["attrs"] == { - "flavor": "cli", - "tool": "loader", - } - assert observed.current_operation() is None - - -def test_segment_with_request_context_does_not_create_implicit_operation( - monkeypatch, -) -> None: - observed = runtime() - observed.segment_duration = RecordingInstrument() - observed.operation_duration = RecordingInstrument() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/calculate", - path="/calculate", - endpoint="calculate", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - monkeypatch.setattr(observed, "current_context", lambda: context) - - with observed.segment(SegmentName.LOAD): - pass - - _, _, attributes = observed.segment_duration.calls[0] - assert attributes["route"] == "/calculate" - assert observed.operation_duration.calls == [] - - -def test_start_scope_outside_request_records_operation_segment_metrics() -> ( - None -): - observed = runtime() - observed.segment_duration = RecordingInstrument() - timings: dict[str, float] = {} - - handle = observed.start_scope( - timings, - name="chat_turn", - flavor="chat", - model="claude", - ) - with observed.segment(SegmentName.LOAD, tool="search"): - pass - observed.end_scope(handle) - - _, _, attributes = observed.segment_duration.calls[0] - assert "load_ms" in timings - assert attributes["operation"] == "chat_turn" - assert attributes["flavor"] == "chat" - assert attributes["model"] == "claude" - assert attributes["tool"] == "search" - - -def test_nested_scope_annotates_span_context_and_operation() -> None: - observed = runtime() - observed.tracer = RecordingTracer() - parent_context = object() - timings: dict[str, float] = {} - - with observed.operation("outer", flavor="chat"): - handle = observed.start_scope( - timings, - name="inner", - parent_context=parent_context, - ) - observed.annotate(handle, model="claude") - observed.mark("custom_ms", 1.23) - observed.mark_ttft() - observed.end_scope(handle) - - span = observed.tracer.span - assert span.attributes["model"] == "claude" - assert "custom_ms" in timings - - -def test_entrypoint_decorator_records_operation_metrics() -> None: - observed = runtime() - observed.operation_duration = RecordingInstrument() - observed.operations = RecordingInstrument() - - @observed.entrypoint("import_data", flavor="cli") - def run_import() -> str: - return "done" - - assert run_import() == "done" - _, _, attributes = observed.operation_duration.calls[0] - assert attributes["operation"] == "import_data" - assert attributes["flavor"] == "cli" - - -def test_async_segment_decorator_records_segment_metrics() -> None: - observed = runtime() - observed.segment_duration = RecordingInstrument() - - @observed.segment(SegmentName.SAVE, flavor="worker") - async def save() -> str: - return "saved" - - assert asyncio.run(save()) == "saved" - _, _, attributes = observed.segment_duration.calls[0] - assert attributes["operation"] == "save" - assert attributes["flavor"] == "worker" - - -def test_record_error_outside_request_uses_operation_context() -> None: - observed = runtime() - observed.errors = RecordingInstrument() - - with observed.operation("worker", flavor="queue"): - observed.record_error( - RuntimeError("failed"), - handled=True, - include_stack=False, - ) - - _, _, attributes = observed.errors.calls[0] - assert attributes["operation"] == "worker" - assert attributes["flavor"] == "queue" - assert attributes["error_type"] == "RuntimeError" - - -def test_record_error_on_request_updates_span_status() -> None: - observed = runtime() - span = RecordingSpan() - observed.trace = SimpleNamespace(get_current_span=lambda: span) - observed.StatusCode = SimpleNamespace(ERROR="ERROR") - observed.Status = lambda code, message: (code, message) - observed.errors = RecordingInstrument() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/error", - path="/error", - endpoint="error", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - observed.begin_request(context) - observed.record_error( - RuntimeError("failed"), - handled=True, - status_code=500, - ) - observed.teardown_request(None) - - assert span.exceptions - assert span.status == ("ERROR", "failed") diff --git a/tests/test_runtime_request_failures.py b/tests/test_runtime_request_failures.py deleted file mode 100644 index 8a22d57..0000000 --- a/tests/test_runtime_request_failures.py +++ /dev/null @@ -1,342 +0,0 @@ -from __future__ import annotations - -from runtime_helpers import ( - RecordingInstrument, - RecordingPropagator, - SegmentName, - runtime, -) - -from policyengine_observability import ( - ObservabilityConfig, - RequestObservabilityContext, -) -from policyengine_observability.config import DEFAULT_METRIC_ATTRIBUTE_KEYS - - -def test_prepare_response_includes_traceparent_when_available() -> None: - observed = runtime() - observed.propagate = RecordingPropagator() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/trace", - path="/trace", - endpoint="trace", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - observed.begin_request(context) - headers = observed.prepare_response(200) - observed.teardown_request(None) - - assert headers["traceparent"].startswith("00-4bf92f") - - -def test_rate_limited_request_records_rate_limit_metric() -> None: - observed = runtime() - observed.rate_limited = RecordingInstrument() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/limited", - path="/limited", - endpoint="limited", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - observed.begin_request(context) - headers = observed.finish_request(429) - observed.teardown_request(None) - - assert headers["X-PolicyEngine-Request-Id"] == "request-1" - assert context.attributes["rate_limited"] is True - assert observed.rate_limited.calls[0][0] == "add" - - -def test_teardown_request_records_unhandled_exception() -> None: - observed = runtime() - observed.errors = RecordingInstrument() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/error", - path="/error", - endpoint="error", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - observed.begin_request(context) - observed.teardown_request(RuntimeError("failed")) - - assert context.status_code == 500 - assert context.error is not None - assert context.error.handled is False - assert observed.errors.calls[0][0] == "add" - - -def test_from_env_invalid_shutdown_timeout_falls_back(monkeypatch) -> None: - monkeypatch.setenv("OBSERVABILITY_SHUTDOWN_TIMEOUT_SECONDS", "bad") - - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.shutdown_timeout_seconds == 3.0 - - -def test_from_env_reads_stdout_format(monkeypatch) -> None: - monkeypatch.setenv("OBSERVABILITY_STDOUT_FORMAT", "google") - - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.stdout_format == "google" - - -def test_from_env_reads_queue_knobs(monkeypatch) -> None: - monkeypatch.setenv("OBSERVABILITY_LOG_QUEUE_MAXSIZE", "50") - monkeypatch.setenv("OBSERVABILITY_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS", "1.5") - - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.log_queue_maxsize == 50 - assert config.log_queue_close_timeout_seconds == 1.5 - - -def test_from_env_queue_knobs_fall_back_on_garbage(monkeypatch) -> None: - monkeypatch.setenv("OBSERVABILITY_LOG_QUEUE_MAXSIZE", "many") - monkeypatch.setenv("OBSERVABILITY_LOG_QUEUE_CLOSE_TIMEOUT_SECONDS", "soon") - - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.log_queue_maxsize == 1000 - assert config.log_queue_close_timeout_seconds == 2.0 - - -def test_from_env_enables_otel_by_default() -> None: - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.otel_enabled is True - - -def test_from_env_allows_otel_opt_out(monkeypatch) -> None: - monkeypatch.setenv("OTEL_ENABLED", "false") - - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.otel_enabled is False - - -def test_from_env_ignores_legacy_observability_otel_switch( - monkeypatch, -) -> None: - monkeypatch.setenv("OBSERVABILITY_OTEL_ENABLED", "false") - - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.otel_enabled is True - - -def test_from_env_reads_boolean_csv_and_environment(monkeypatch) -> None: - monkeypatch.setenv("OBSERVABILITY_SERVICE_NAME", "env-svc") - monkeypatch.setenv("DEPLOYMENT_ENVIRONMENT", "production") - monkeypatch.setenv("OBSERVABILITY_ENABLED", "off") - monkeypatch.setenv("OBSERVABILITY_REQUEST_LOGS_ENABLED", "false") - monkeypatch.setenv("OBSERVABILITY_LOG_RAW_IP", "0") - monkeypatch.setenv("OBSERVABILITY_LOG_LEVEL", "warning") - monkeypatch.setenv("OTEL_ENABLED", "1") - monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector") - monkeypatch.setenv("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf") - monkeypatch.setenv("OBSERVABILITY_TRACER_NAME", "tracer") - monkeypatch.setenv("OBSERVABILITY_METER_NAME", "meter") - monkeypatch.setenv( - "OBSERVABILITY_METRIC_ATTRIBUTE_KEYS", - "service.name, custom", - ) - monkeypatch.setenv( - "OBSERVABILITY_EXTRA_METRIC_ATTRIBUTE_KEYS", - "custom, other", - ) - - config = ObservabilityConfig.from_env( - service_name="svc", - instrument_fastapi=True, - instrument_httpx=True, - ) - - assert config.service_name == "env-svc" - assert config.environment == "production" - assert config.enabled is False - assert config.request_logs_enabled is False - assert config.log_raw_ip is False - assert config.otel_enabled is True - assert config.otlp_endpoint == "http://collector" - assert config.otlp_protocol == "http/protobuf" - assert config.tracer_name == "tracer" - assert config.meter_name == "meter" - assert config.instrument_fastapi is True - assert config.instrument_httpx is True - assert config.metric_attribute_keys == ("service.name", "custom", "other") - - -def test_from_env_reads_log_destinations_and_google_config( - monkeypatch, -) -> None: - monkeypatch.setenv( - "OBSERVABILITY_LOG_DESTINATIONS", - "stdout, google-cloud-logging, stdout", - ) - monkeypatch.setenv("GCP_PROJECT", "fallback-project") - monkeypatch.setenv("OBSERVABILITY_GOOGLE_CLOUD_LOG_NAME", "custom-log") - - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.log_destinations == ("stdout", "google-cloud-logging") - assert config.google_cloud_project == "fallback-project" - assert config.google_cloud_log_name == "custom-log" - - -def test_from_env_uses_default_log_destinations_without_env( - monkeypatch, -) -> None: - monkeypatch.delenv("OBSERVABILITY_LOG_DESTINATIONS", raising=False) - - config = ObservabilityConfig.from_env( - service_name="svc", - default_log_destinations=("google_cloud_logging",), - ) - - assert config.log_destinations == ("google_cloud_logging",) - - -def test_from_env_log_destinations_env_overrides_default( - monkeypatch, -) -> None: - monkeypatch.setenv("OBSERVABILITY_LOG_DESTINATIONS", "stdout") - - config = ObservabilityConfig.from_env( - service_name="svc", - default_log_destinations=("google_cloud_logging",), - ) - - assert config.log_destinations == ("stdout",) - - -def test_metric_attribute_keys_are_configurable() -> None: - config = ObservabilityConfig( - service_name="svc", - metric_attribute_keys=("service.name", "tool"), - ) - context = RequestObservabilityContext( - config=config, - request_id="request-1", - method="POST", - route="/chat", - path="/chat", - endpoint="chat", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - context.set_attribute("tool", "search") - context.set_attribute("model", "claude") - - assert context.metric_attributes() == { - "service.name": "svc", - "tool": "search", - } - - -def test_context_set_attribute_normalizes_enum_values() -> None: - observed = runtime() - operation = observed.start_operation("job")["operation"] - request = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/", - path="/", - endpoint="root", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - operation.set_attribute("segment", SegmentName.LOAD) - request.set_attribute("segment", SegmentName.SAVE) - observed.end_operation({"operation": operation}) - - assert operation.attributes["segment"] == "load" - assert request.attributes["segment"] == "save" - - -def test_metric_attribute_keys_can_be_extended_from_env(monkeypatch) -> None: - monkeypatch.setenv("OBSERVABILITY_EXTRA_METRIC_ATTRIBUTE_KEYS", "custom") - - config = ObservabilityConfig.from_env(service_name="svc") - - assert config.metric_attribute_keys == ( - *DEFAULT_METRIC_ATTRIBUTE_KEYS, - "custom", - ) - - -def test_segment_metric_uses_configured_metric_attribute_keys() -> None: - observed = runtime( - metric_attribute_keys=( - "service.name", - "route", - "method", - "segment", - "tool", - ) - ) - observed.segment_duration = RecordingInstrument() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="POST", - route="/chat", - path="/chat", - endpoint="chat", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - observed.begin_request(context) - with observed.segment(SegmentName.LOAD, tool="search", model="claude"): - pass - observed.finish_request(200) - observed.teardown_request(None) - - _, _, attributes = observed.segment_duration.calls[0] - assert attributes["tool"] == "search" - assert "model" not in attributes - - -def test_shutdown_calls_trace_and_metric_providers() -> None: - class Provider: - def __init__(self) -> None: - self.shutdown_called = False - - def shutdown(self) -> None: - self.shutdown_called = True - - observed = runtime(shutdown_timeout_seconds=1) - trace_provider = Provider() - meter_provider = Provider() - observed.tracer_provider = trace_provider - observed.meter_provider = meter_provider - - observed.shutdown() - - assert trace_provider.shutdown_called - assert meter_provider.shutdown_called diff --git a/tests/test_runtime_requests.py b/tests/test_runtime_requests.py deleted file mode 100644 index fb42d62..0000000 --- a/tests/test_runtime_requests.py +++ /dev/null @@ -1,331 +0,0 @@ -from __future__ import annotations - -from contextvars import ContextVar -from types import SimpleNamespace - -from runtime_helpers import ( - NamedRecordingSpan, - RecordingInstrument, - SegmentName, - runtime, -) - -from policyengine_observability import ( - RequestObservabilityContext, -) -from policyengine_observability import _state as state_module - - -def test_request_lifecycle_records_headers_and_context_metrics() -> None: - observed = runtime() - observed.active_requests = RecordingInstrument() - observed.requests = RecordingInstrument() - observed.http_duration = RecordingInstrument() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/calculate", - path="/calculate", - endpoint="calculate", - query_keys=["country"], - content_length_bytes=None, - inbound={"ip_source": "remote_addr", "client_ip": "127.0.0.1"}, - ) - - observed.begin_request(context) - with observed.segment(SegmentName.LOAD): - pass - headers = observed.finish_request(200) - observed.teardown_request(None) - - assert headers["X-PolicyEngine-Request-Id"] == "request-1" - assert context.status_code == 200 - assert "load" in context.timings_ms - assert observed.current_context() is None - assert observed.current_operation() is None - assert observed.active_requests.calls[0][1] == 1 - assert observed.active_requests.calls[-1][1] == -1 - assert observed.requests.calls[0][1] == 1 - - -def test_request_log_accumulates_repeated_segment_timings() -> None: - observed = runtime() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/calculate", - path="/calculate", - endpoint="calculate", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - observed.begin_request(context) - with observed.segment(SegmentName.LOAD): - pass - with observed.segment(SegmentName.LOAD): - pass - observed.finish_request(200) - observed.teardown_request(None) - - assert context.timings_ms["load"] >= 0 - assert context.timing_counts["load"] == 2 - payload = context.as_log_record(trace_id=None, span_id=None) - assert payload["timing_counts"]["load"] == 2 - assert [node["name"] for node in payload["segment_tree"]] == [ - "load", - "load", - ] - - -def test_internal_dispatch_segments_merge_into_parent_operation() -> None: - observed = runtime() - handle = observed.start_operation( - "modal_worker_dispatch", - flavor="modal_worker", - ) - parent_operation = handle["operation"] - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="POST", - route="/calculate", - path="/calculate", - endpoint="calculate", - query_keys=[], - content_length_bytes=None, - inbound={}, - internal_dispatch=True, - ) - - try: - observed.begin_request(context) - with observed.segment(SegmentName.LOAD): - pass - observed.finish_request(200) - observed.teardown_request(None) - - assert context.timings_ms is parent_operation.timings_ms - assert context.timing_counts is parent_operation.timing_counts - assert context.segment_tree is parent_operation.segment_tree - assert "load" in parent_operation.timings_ms - assert parent_operation.timing_counts["load"] == 1 - assert parent_operation.segment_tree[0].name == "load" - assert observed.current_operation() is parent_operation - finally: - observed.end_operation(handle) - - assert observed.current_context() is None - assert observed.current_operation() is None - - -def test_non_internal_request_timings_do_not_leak_to_parent_operation() -> ( - None -): - observed = runtime() - handle = observed.start_operation("job", flavor="worker") - parent_operation = handle["operation"] - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="POST", - route="/calculate", - path="/calculate", - endpoint="calculate", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - try: - observed.begin_request(context) - with observed.segment(SegmentName.LOAD): - pass - observed.finish_request(200) - observed.teardown_request(None) - - assert context.timings_ms is not parent_operation.timings_ms - assert context.segment_tree is not parent_operation.segment_tree - assert "load" not in parent_operation.timings_ms - assert parent_operation.segment_tree == [] - assert context.segment_tree[0].name == "load" - assert observed.current_operation() is parent_operation - finally: - observed.end_operation(handle) - - assert observed.current_context() is None - assert observed.current_operation() is None - - -def test_set_attribute_updates_explicit_operation_inside_request() -> None: - observed = runtime() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/chat", - path="/chat", - endpoint="chat", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - observed.begin_request(context) - handle = observed.start_operation("chat.turn", flavor="chat") - operation = handle["operation"] - try: - observed.set_attribute("model", "claude") - finally: - observed.end_operation(handle) - observed.teardown_request(None) - - assert context.attributes["model"] == "claude" - assert context.operation_context.attributes["model"] == "claude" - assert operation.attributes["model"] == "claude" - assert observed.current_context() is None - assert observed.current_operation() is None - - -def test_mark_ttft_attribute_updates_current_operation() -> None: - observed = runtime() - handle = observed.start_operation("chat.turn", flavor="chat") - operation = handle["operation"] - - try: - observed.mark_ttft_attribute() - finally: - observed.end_operation(handle) - - assert operation.attributes["ttft_ms"] >= 0 - - -def test_request_methods_noop_without_current_context() -> None: - observed = runtime() - - assert observed.prepare_response(200) == {} - observed.complete_request(200) - observed.update_request_route(route="/missing") - observed.teardown_request(None) - - -def test_request_begin_operation_begin_and_lifecycle_failures_are_logged( - monkeypatch, -) -> None: - class BrokenVar: - def set(self, _value): - raise RuntimeError("set failed") - - observed = runtime() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/broken", - path="/broken", - endpoint="broken", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - monkeypatch.setattr(state_module, "_REQUEST_CONTEXT", BrokenVar()) - observed.begin_request(context) - monkeypatch.setattr( - state_module, - "_REQUEST_CONTEXT", - ContextVar("request", default=None), - ) - monkeypatch.setattr(state_module, "_OPERATION_CONTEXT", BrokenVar()) - observed._begin_request_operation(context) - - assert failures == ["request.begin", "request.operation_begin"] - - -def test_request_prepare_complete_update_and_teardown_failures_are_logged() -> ( - None -): - observed = runtime() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/broken", - path="/broken", - endpoint="broken", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - observed.begin_request(context) - context.span_attributes = lambda **_extra: (_ for _ in ()).throw( - RuntimeError("span attrs failed") - ) - observed.prepare_response(200) - observed.complete_request(200) - observed.update_request_route(route="/other") - observed.emit_request_log = lambda _context: (_ for _ in ()).throw( - RuntimeError("emit failed") - ) - observed.teardown_request(None) - - assert "request.prepare_response" in failures - assert "request.complete" in failures - assert "request.update_route" in failures - assert "request.teardown" in failures - - -def test_set_attribute_failure_path_is_logged() -> None: - observed = runtime() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - observed.current_context = lambda: SimpleNamespace( - set_attribute=lambda *_args: (_ for _ in ()).throw( - RuntimeError("attribute failed") - ) - ) - - observed.set_attribute("tool", "loader") - - assert failures == ["request.set_attribute"] - - -def test_request_route_update_relabels_active_request_and_span() -> None: - observed = runtime() - observed.active_requests = RecordingInstrument() - span = NamedRecordingSpan() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/initial", - path="/items/1", - endpoint="initial", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - context.server_span = span - - observed.begin_request(context) - observed.update_request_route(route="/items/", endpoint="item") - observed.teardown_request(None) - - assert context.route == "/items/" - assert context.endpoint == "item" - assert span.names == ["/items/"] - assert observed.active_requests.calls[1][1] == -1 - assert observed.active_requests.calls[2][1] == 1 diff --git a/tests/test_runtime_segments.py b/tests/test_runtime_segments.py deleted file mode 100644 index c92021a..0000000 --- a/tests/test_runtime_segments.py +++ /dev/null @@ -1,303 +0,0 @@ -from __future__ import annotations - -import asyncio -from typing import Any - -import pytest -from runtime_helpers import ( - AttributeFailingSpan, - RecordingSpan, - RecordingTracer, - SegmentName, - runtime, -) - -from policyengine_observability import ( - UNKNOWN_SEGMENT, - ObservabilityRuntime, - RequestObservabilityContext, -) - - -def test_segment_records_aggregated_timing() -> None: - observed = runtime() - - with observed.collect_timings("request") as timings: - with observed.segment(SegmentName.LOAD): - pass - with observed.segment(SegmentName.LOAD): - pass - - assert "load_ms" in timings - assert timings["load_ms"] >= 0 - - -def test_operation_log_accumulates_repeated_segment_timings() -> None: - observed = runtime() - handle = observed.start_operation("job") - operation = handle["operation"] - - try: - with observed.segment(SegmentName.LOAD): - pass - with observed.segment(SegmentName.LOAD): - pass - finally: - observed.end_operation(handle) - - assert operation.timings_ms["load"] >= 0 - assert operation.timing_counts["load"] == 2 - payload = operation.as_log_record(trace_id=None, span_id=None) - assert payload["timing_counts"]["load"] == 2 - - -def test_operation_log_records_ordered_nested_segment_tree() -> None: - observed = runtime() - handle = observed.start_operation("job") - operation = handle["operation"] - - try: - with observed.segment(SegmentName.LOAD): - with observed.segment( - SegmentName.SAVE, - simulation_kind="baseline", - token="SECRET", - payload={"not": "safe"}, - ): - pass - with observed.segment( - SegmentName.SAVE, - simulation_kind="reform", - ): - pass - finally: - observed.end_operation(handle) - - payload = operation.as_log_record(trace_id=None, span_id=None) - tree = payload["segment_tree"] - assert len(tree) == 1 - assert tree[0]["sequence"] == 1 - assert tree[0]["name"] == "load" - assert "duration_ms" in tree[0] - assert "self_ms" not in tree[0] - - children = tree[0]["children"] - assert [child["sequence"] for child in children] == [2, 3] - assert [child["name"] for child in children] == ["save", "save"] - assert children[0]["attrs"] == {"simulation_kind": "baseline"} - assert children[1]["attrs"] == {"simulation_kind": "reform"} - assert "token" not in children[0].get("attrs", {}) - assert "payload" not in children[0].get("attrs", {}) - assert payload["timing_counts"]["save"] == 2 - - -def test_operation_log_reserved_fields_override_attributes() -> None: - observed = runtime() - handle = observed.start_operation( - "job", - operation="attribute-operation", - duration_ms="attribute-duration", - timings_ms="attribute-timings", - timing_counts="attribute-counts", - segment_tree="attribute-tree", - error="attribute-error", - ) - operation = handle["operation"] - - try: - with observed.segment(SegmentName.LOAD): - pass - finally: - observed.end_operation(handle) - - payload = operation.as_log_record(trace_id=None, span_id=None) - assert payload["operation"] == "job" - assert isinstance(payload["duration_ms"], float) - assert isinstance(payload["timings_ms"], dict) - assert isinstance(payload["timing_counts"], dict) - assert isinstance(payload["segment_tree"], list) - assert payload["error"] is None - - -def test_async_segment_records_timing() -> None: - async def run() -> dict[str, float]: - observed = runtime() - with observed.collect_timings("request") as timings: - async with observed.asegment(SegmentName.SAVE): - pass - return timings - - timings = asyncio.run(run()) - - assert "save_ms" in timings - - -def test_async_segments_keep_independent_segment_tree_stacks() -> None: - async def run() -> list[dict[str, Any]]: - observed = runtime() - handle = observed.start_operation("job") - operation = handle["operation"] - - async def branch(branch_name: str) -> None: - async with observed.asegment(SegmentName.LOAD, branch=branch_name): - await asyncio.sleep(0) - async with observed.asegment( - SegmentName.SAVE, - branch=branch_name, - ): - await asyncio.sleep(0) - - try: - await asyncio.gather(branch("a"), branch("b")) - finally: - observed.end_operation(handle) - return operation.as_log_record(trace_id=None, span_id=None)[ - "segment_tree" - ] - - tree = asyncio.run(run()) - - assert [node["name"] for node in tree] == ["load", "load"] - assert [node["attrs"] for node in tree] == [ - {"branch": "a"}, - {"branch": "b"}, - ] - assert [node["children"][0]["attrs"] for node in tree] == [ - {"branch": "a"}, - {"branch": "b"}, - ] - - -def test_segment_preserves_business_exception_and_records_timing() -> None: - observed = runtime() - - with pytest.raises(ValueError, match="business failed"): - with observed.collect_timings("request") as timings: - with observed.segment(SegmentName.LOAD): - raise ValueError("business failed") - - assert "load_ms" in timings - - -def test_segment_tree_records_failed_segments_before_reraising() -> None: - observed = runtime() - handle = observed.start_operation("job") - operation = handle["operation"] - error = None - - try: - with observed.segment(SegmentName.LOAD): - raise ValueError("business failed") - except ValueError as exc: - error = exc - finally: - observed.end_operation(handle, error) - - payload = operation.as_log_record(trace_id=None, span_id=None) - assert payload["event"] == "operation_failed" - assert payload["segment_tree"][0]["name"] == "load" - assert "duration_ms" in payload["segment_tree"][0] - - -def test_unregistered_segment_falls_back_without_throwing() -> None: - class BrokenString: - def __str__(self) -> str: - raise RuntimeError("cannot stringify") - - observed = runtime() - - with observed.collect_timings("request") as timings: - with observed.segment(BrokenString()): - pass - - assert f"{UNKNOWN_SEGMENT}_ms" in timings - - -def test_segment_span_start_failure_does_not_skip_user_code() -> None: - observed = runtime() - observed.tracer = RecordingTracer(fail_enter=True) - executed = False - - with observed.collect_timings("request") as timings: - with observed.segment(SegmentName.LOAD): - executed = True - - assert executed - assert "load_ms" in timings - - -def test_segment_span_exit_failure_does_not_escape() -> None: - observed = runtime() - observed.tracer = RecordingTracer(fail_exit=True) - - with observed.collect_timings("request") as timings: - with observed.segment(SegmentName.LOAD): - pass - - assert "load_ms" in timings - - -def test_disabled_runtime_noops_across_public_methods() -> None: - observed = ObservabilityRuntime.disabled() - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/disabled", - path="/disabled", - endpoint="disabled", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - handle = observed.start_operation("disabled") - observed.end_operation(handle) - observed.begin_request(context) - observed.complete_request(200) - observed.update_request_route(route="/other") - observed.teardown_request(None) - observed.set_attribute("key", "value") - observed.record_error(RuntimeError("ignored"), handled=True) - observed.record_event("ignored") - - with observed.segment(SegmentName.LOAD) as span: - assert span is None - - async def run() -> None: - async with observed.asegment(SegmentName.LOAD) as async_span: - assert async_span is None - - asyncio.run(run()) - assert observed.prepare_response(200) == {} - assert observed.current_context() is None - assert observed.current_operation() is None - - -def test_span_attribute_failure_does_not_drop_span_lifecycle() -> None: - observed = runtime() - span = AttributeFailingSpan() - observed.tracer = RecordingTracer(span=span) - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - - with observed.segment(SegmentName.LOAD, tool="loader"): - pass - - assert "otel.span_attributes" in failures - assert observed.tracer.last_context_manager.exited - - -def test_collect_timings_records_block_exception_on_scope_span() -> None: - observed = runtime() - span = RecordingSpan() - observed.tracer = RecordingTracer(span=span) - - with pytest.raises(RuntimeError, match="scope failed"): - with observed.collect_timings("turn"): - raise RuntimeError("scope failed") - - assert len(span.exceptions) == 1 - assert isinstance(span.exceptions[0], RuntimeError) diff --git a/tests/test_runtime_tracing.py b/tests/test_runtime_tracing.py deleted file mode 100644 index cef5cf3..0000000 --- a/tests/test_runtime_tracing.py +++ /dev/null @@ -1,420 +0,0 @@ -from __future__ import annotations - -import builtins -import time -from types import SimpleNamespace - -import pytest -from runtime_helpers import ( - AttributeFailingSpan, - ExceptionFailingSpan, - RecordingMeter, - RecordingPropagator, - RecordingTracer, - ValidContextSpan, - runtime, -) - -from policyengine_observability import ( - RequestObservabilityContext, -) -from policyengine_observability import _state as state_module -from policyengine_observability import runtime as runtime_module - - -def test_shutdown_logs_provider_failures_and_timeout() -> None: - class FailingProvider: - def shutdown(self) -> None: - raise RuntimeError("shutdown failed") - - class SlowProvider: - def shutdown(self) -> None: - time.sleep(0.05) - - observed = runtime(shutdown_timeout_seconds=0.001) - observed.tracer_provider = FailingProvider() - observed.meter_provider = SlowProvider() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - - observed.shutdown() - - assert "otel.trace_shutdown" in failures - assert "otel.shutdown_timeout" in failures - - -def test_shutdown_closes_destinations_with_full_budget_and_no_thread( - monkeypatch, -) -> None: - observed = runtime(shutdown_timeout_seconds=2.0) - close_calls = [] - monkeypatch.setattr( - observed.log_destination_manager, - "close", - lambda deadline=None: close_calls.append(deadline), - ) - - def fail_thread(*args, **kwargs): - raise AssertionError( - "no watchdog thread should exist without providers" - ) - - monkeypatch.setattr(runtime_module.threading, "Thread", fail_thread) - - observed.shutdown() - - assert close_calls == [2.0] - - -def test_shutdown_destination_deadline_fits_inside_provider_budget( - monkeypatch, -) -> None: - """The prior design gave the log flush a deadline larger than the - join bounding it, starving provider shutdown; the deadline must be - derived from (and smaller than) the shutdown budget.""" - - class Provider: - def __init__(self) -> None: - self.shutdown_called = False - - def shutdown(self) -> None: - self.shutdown_called = True - - observed = runtime(shutdown_timeout_seconds=2.0) - provider = Provider() - observed.tracer_provider = provider - close_calls = [] - monkeypatch.setattr( - observed.log_destination_manager, - "close", - lambda deadline=None: close_calls.append(deadline), - ) - - observed.shutdown() - - assert close_calls == [1.0] - assert provider.shutdown_called - - -def test_shutdown_slow_destination_close_still_runs_providers() -> None: - class Provider: - def __init__(self) -> None: - self.shutdown_called = False - - def shutdown(self) -> None: - self.shutdown_called = True - - observed = runtime(shutdown_timeout_seconds=0.2) - provider = Provider() - observed.tracer_provider = provider - observed.log_destination_manager.close = lambda deadline=None: time.sleep( - 0.05 - ) - - observed.shutdown() - - assert provider.shutdown_called - - -def test_shutdown_clamps_pathological_budget(monkeypatch) -> None: - observed = runtime(shutdown_timeout_seconds=float("inf")) - close_calls = [] - monkeypatch.setattr( - observed.log_destination_manager, - "close", - lambda deadline=None: close_calls.append(deadline), - ) - - observed.shutdown() - - assert close_calls == [3.0] - - -def test_restart_log_destinations_rebuilds_from_config() -> None: - observed = runtime(otel_enabled=False) - observed.configure() - first = observed.log_destination_manager.destinations[0] - - observed.restart_log_destinations() - - rebuilt = observed.log_destination_manager.destinations - assert len(rebuilt) == 1 - assert rebuilt[0] is not first - assert observed.log_destination_manager.configured is True - - -def test_restart_log_destinations_noops_when_disabled(monkeypatch) -> None: - """The kill switch must hold across forks and snapshot restores: - a disabled runtime's restart must not build destinations.""" - observed = runtime(enabled=False) - configure_calls = [] - monkeypatch.setattr( - observed.log_destination_manager, - "configure", - lambda: configure_calls.append(True), - ) - - observed.restart_log_destinations() - - assert configure_calls == [] - - -def test_shutdown_survives_destination_close_failure(monkeypatch) -> None: - class Provider: - def __init__(self) -> None: - self.shutdown_called = False - - def shutdown(self) -> None: - self.shutdown_called = True - - observed = runtime(shutdown_timeout_seconds=1.0) - provider = Provider() - observed.tracer_provider = provider - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - - def broken_close(deadline=None): - raise RuntimeError("close exploded") - - monkeypatch.setattr( - observed.log_destination_manager, "close", broken_close - ) - - observed.shutdown() - - assert provider.shutdown_called - assert "logging.destination_close" in failures - - -def test_configure_otel_creates_real_providers_and_instruments() -> None: - observed = runtime(otel_enabled=True) - - observed.configure() - - assert observed.tracer is not None - assert observed.meter is not None - assert observed.trace is not None - assert observed.propagate is not None - - -def test_configure_otel_with_exporters_does_not_throw() -> None: - observed = runtime( - otel_enabled=True, - otlp_endpoint="http://localhost:4318", - otlp_protocol="http/protobuf", - ) - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - - observed.configure() - - assert observed.tracer_provider is not None - assert observed.meter_provider is not None - - -def test_configure_otel_import_failure_is_logged(monkeypatch) -> None: - observed = runtime(otel_enabled=True) - failures = [] - original_import = builtins.__import__ - - def failing_import(name, *args, **kwargs): - if name == "opentelemetry": - raise RuntimeError("otel missing") - return original_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", failing_import) - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - - observed.configure() - - assert failures == ["otel.configure_imports"] - - -def test_configure_instruments_and_instrument_failures() -> None: - observed = runtime() - meter = RecordingMeter() - observed.meter = meter - - observed._configure_instruments() - - assert len(meter.created) == 11 - - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append((operation, fields.get("instrument"))) - ) - noop = observed._instrument( - lambda *_args, **_kwargs: (_ for _ in ()).throw( - RuntimeError("factory failed") - ), - "broken", - ) - - noop.add(1) - noop.record(1) - assert failures == [("metrics.create_instrument", "broken")] - - -def test_request_span_lifecycle_records_enter_and_exit_failures() -> None: - observed = runtime() - observed.tracer = RecordingTracer(fail_enter=True) - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - context = RequestObservabilityContext( - config=observed.config, - request_id="request-1", - method="GET", - route="/span", - path="/span", - endpoint="span", - query_keys=[], - content_length_bytes=None, - inbound={}, - ) - - observed._start_request_span(context) - assert context.server_span is None - - observed.tracer = RecordingTracer(fail_exit=True) - observed._start_request_span(context) - observed._close_request_span(context, RuntimeError("failed")) - observed._close_request_span(context, None) - - assert failures == ["otel.request_span_enter", "otel.request_span_exit"] - - -def test_safe_span_records_exception_and_preserves_user_error() -> None: - observed = runtime() - observed.tracer = RecordingTracer() - - with pytest.raises(RuntimeError, match="business failed"): - with observed._safe_span("safe", {}): - raise RuntimeError("business failed") - - assert isinstance(observed.tracer.span.exceptions[0], RuntimeError) - - -def test_span_and_segment_failure_helpers_are_logged(monkeypatch) -> None: - observed = runtime() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - observed.trace = SimpleNamespace( - get_current_span=lambda: AttributeFailingSpan() - ) - observed._set_current_span_attributes({"key": "value"}) - - observed.trace = SimpleNamespace( - get_current_span=lambda: ExceptionFailingSpan() - ) - observed._record_exception_on_span( - ExceptionFailingSpan(), - RuntimeError("failed"), - handled=False, - status_code=500, - ) - observed._add_span_event("event", {"safe": "yes", "unsafe": object()}) - observed._record_segment_safely("missing_start", None, {}) - monkeypatch.setattr( - observed, - "_safe_perf_counter", - lambda _operation: None, - ) - observed._record_segment_safely("missing_end", 1.0, {}) - - assert "otel.set_span_attributes" in failures - assert "otel.record_exception" in failures - - -def test_segment_helpers_cover_operation_attrs_and_span_prefix() -> None: - observed = runtime(span_prefix="svc") - with observed.operation("job", flavor="cli"): - attrs = observed._segment_span_attributes({"tool": "loader"}) - - assert attrs["policyengine.operation"] == "job" - assert attrs["tool"] == "loader" - assert observed._span_name("load") == "svc.load" - - -def test_contextvar_failure_paths_are_logged(monkeypatch) -> None: - class BrokenVar: - def get(self): - raise RuntimeError("get failed") - - observed = runtime() - failures = [] - observed.log_observability_failure = lambda operation, exc, **fields: ( - failures.append(operation) - ) - monkeypatch.setattr(state_module, "_REQUEST_CONTEXT", BrokenVar()) - monkeypatch.setattr(state_module, "_OPERATION_CONTEXT", BrokenVar()) - - assert observed.current_context() is None - assert observed.current_operation() is None - assert failures == ["context.current", "operation.current"] - - -def test_runtime_owned_httpx_instrumentation_failure_does_not_throw( - monkeypatch, -) -> None: - observed = runtime(otel_enabled=True) - failures = [] - original_import = builtins.__import__ - - def failing_import(name, *args, **kwargs): - if name == "opentelemetry.instrumentation.httpx": - raise RuntimeError("instrumentation failed") - return original_import(name, *args, **kwargs) - - monkeypatch.setattr(builtins, "__import__", failing_import) - monkeypatch.setattr( - observed, - "log_observability_failure", - lambda operation, exc, **fields: failures.append(operation), - ) - - observed.instrument_httpx() - - assert failures == ["httpx.auto_instrument"] - - -def test_runtime_owned_httpx_instrumentation_success_and_wrapper() -> None: - from policyengine_observability.integrations.httpx import ( - instrument_httpx, - ) - - observed = runtime(otel_enabled=True) - - instrument_httpx(observed) - instrument_httpx(observed) - - assert observed._httpx_instrumented is True - - -def test_traceparent_capture_and_valid_trace_ids() -> None: - observed = runtime() - propagator = RecordingPropagator() - observed.propagate = propagator - span = ValidContextSpan() - observed.trace = SimpleNamespace(get_current_span=lambda: span) - - trace_id, span_id = observed._trace_ids() - - assert observed.traceparent_header().startswith("00-4bf92f") - assert observed._extract_context({"traceparent": "parent"}) == { - "parent": {"traceparent": "parent"} - } - assert propagator.extracted == {"traceparent": "parent"} - assert trace_id == "4bf92f3577b34da6a3ce929d0e0e4736" - assert span_id == "00f067aa0ba902b7" diff --git a/tests/test_stdout_destinations.py b/tests/test_stdout_destinations.py deleted file mode 100644 index aef9c4c..0000000 --- a/tests/test_stdout_destinations.py +++ /dev/null @@ -1,362 +0,0 @@ -from __future__ import annotations - -import json - -from fakes import ( - ClosableRecordingDestination, - FailingDestination, - RecordingDestination, - RecordingLogger, - make_manager, -) - -from policyengine_observability.config import ObservabilityConfig -from policyengine_observability.destinations.stdout import ( - StdoutJsonDestination, - resolve_stdout_formatter, -) - - -def _stdout_destination(config=None, formatter=None): - logger = RecordingLogger() - if formatter is None and config is not None: - formatter = resolve_stdout_formatter(config) - destination = StdoutJsonDestination( - loggers={"event": logger}, - serializer=json.dumps, - formatter=formatter, - ) - return destination, logger - - -def _emitted_line(logger): - ((_, message),) = logger.lines - return json.loads(message) - - -def test_stdout_google_formatter_maps_agent_native_keys() -> None: - config = ObservabilityConfig( - stdout_format="google", google_cloud_project="proj" - ) - destination, logger = _stdout_destination(config) - - destination.emit( - { - "schema_version": "policyengine.observability.event.v1", - "service_name": "svc", - "event": "x", - "trace_id": "abc123", - "span_id": 456, - }, - log_type="event", - severity="error", - ) - - line = _emitted_line(logger) - assert line["severity"] == "ERROR" - assert line["logging.googleapis.com/trace"] == ( - "projects/proj/traces/abc123" - ) - assert line["logging.googleapis.com/spanId"] == "456" - assert line["logging.googleapis.com/labels"] == { - "log_type": "event", - "service_name": "svc", - "schema_version": "policyengine.observability.event.v1", - } - assert "time" not in line - assert line["event"] == "x" - - -def test_stdout_google_formatter_omits_trace_without_project() -> None: - config = ObservabilityConfig(stdout_format="google") - destination, logger = _stdout_destination(config) - - destination.emit( - {"event": "x", "trace_id": "abc"}, log_type="event", severity="INFO" - ) - - line = _emitted_line(logger) - assert "logging.googleapis.com/trace" not in line - - -def test_stdout_unknown_format_falls_back_to_plain() -> None: - config = ObservabilityConfig(stdout_format=" GoOgLeX ") - destination, logger = _stdout_destination(config) - - destination.emit({"event": "x"}, log_type="event", severity="INFO") - - assert _emitted_line(logger) == {"event": "x"} - - -def test_stdout_unknown_format_reports_when_channel_available() -> None: - failures = [] - formatter = resolve_stdout_formatter( - ObservabilityConfig(stdout_format="agent-natve"), - on_failure=lambda operation, exc, **fields: failures.append( - (operation, str(exc)) - ), - ) - - formatted = formatter({"event": "x"}, log_type="event", severity="INFO") - - assert formatted == {"event": "x"} - assert len(failures) == 1 - assert failures[0][0] == "logging.stdout_format" - assert "agent-natve" in failures[0][1] - - -def test_stdout_format_name_is_normalized() -> None: - config = ObservabilityConfig(stdout_format=" GOOGLE ") - destination, logger = _stdout_destination(config) - - destination.emit({"event": "x"}, log_type="event", severity="INFO") - - assert _emitted_line(logger)["severity"] == "INFO" - - -def test_stdout_broken_formatter_degrades_to_unformatted() -> None: - def broken(payload, *, log_type, severity): - raise RuntimeError("formatter bug") - - destination, logger = _stdout_destination(formatter=broken) - - destination.emit({"event": "x"}, log_type="event", severity="INFO") - - assert _emitted_line(logger) == {"event": "x"} - - -def test_stdout_broken_formatter_factory_degrades_to_plain() -> None: - from policyengine_observability.destinations.stdout import ( - _FORMATTER_FACTORIES, - register_stdout_formatter, - ) - - def broken_factory(config): - raise RuntimeError("factory bug") - - register_stdout_formatter("broken-test", broken_factory) - try: - failures = [] - formatter = resolve_stdout_formatter( - ObservabilityConfig(stdout_format="broken-test"), - on_failure=lambda operation, exc, **fields: failures.append( - (operation, fields) - ), - ) - finally: - _FORMATTER_FACTORIES.pop("broken_test", None) - - formatted = formatter({"event": "x"}, log_type="event", severity="INFO") - - assert formatted == {"event": "x"} - assert len(failures) == 1 - assert failures[0][0] == "logging.stdout_format" - assert failures[0][1]["stdout_format"] == "broken-test" - - -def test_configure_fallback_survives_broken_formatter_factory() -> None: - """The crash path: no destination builds, so the manager's - last-resort stdout fallback resolves the same broken formatter — - configure must stay fail-open and emit must still write plain.""" - from policyengine_observability.destinations.stdout import ( - _FORMATTER_FACTORIES, - register_stdout_formatter, - ) - - def broken_factory(config): - raise RuntimeError("factory bug") - - register_stdout_formatter("broken-test", broken_factory) - try: - config = ObservabilityConfig( - log_destinations=("nonexistent",), stdout_format="broken-test" - ) - logger = RecordingLogger() - manager, _failures = make_manager(config, loggers={"event": logger}) - - manager.configure() # raised before the resolver guard existed - manager.emit({"event": "x"}, log_type="event", severity="INFO") - finally: - _FORMATTER_FACTORIES.pop("broken_test", None) - - assert _emitted_line(logger) == {"event": "x", "severity": "INFO"} - - -def test_stdout_google_formatter_never_mutates_caller_payload() -> None: - config = ObservabilityConfig( - stdout_format="google", google_cloud_project="proj" - ) - destination, logger = _stdout_destination(config) - payload = {"event": "x", "trace_id": "abc"} - - destination.emit(payload, log_type="event", severity="INFO") - - assert payload == {"event": "x", "trace_id": "abc"} - - -def test_custom_stdout_formatter_registers_and_resolves() -> None: - from policyengine_observability.destinations.stdout import ( - _FORMATTER_FACTORIES, - register_stdout_formatter, - ) - - def factory(config): - return lambda payload, *, log_type, severity: {"wrapped": payload} - - register_stdout_formatter("custom-test", factory) - try: - # Hyphen/underscore variance is forgiven the same way it is for - # destination names. - config = ObservabilityConfig(stdout_format=" Custom_Test ") - destination, logger = _stdout_destination(config) - destination.emit({"event": "x"}, log_type="event", severity="INFO") - finally: - _FORMATTER_FACTORIES.pop("custom_test", None) - - assert _emitted_line(logger) == {"wrapped": {"event": "x"}} - - -# ── Destination circuit breaker and manager lifecycle ─────────────────── - - -def test_destination_disabled_after_consecutive_emit_failures() -> None: - from policyengine_observability.destinations.manager import ( - DESTINATION_FAILURE_LIMIT, - ) - - flaky = FailingDestination() - healthy = RecordingDestination() - manager, failures = make_manager(destinations=[flaky, healthy]) - - for _ in range(DESTINATION_FAILURE_LIMIT + 2): - manager.emit({"event": "x"}, log_type="event", severity="INFO") - - assert flaky.calls == DESTINATION_FAILURE_LIMIT - assert flaky not in manager.destinations - assert len(healthy.payloads) == DESTINATION_FAILURE_LIMIT + 2 - assert any(op == "logging.destination_disabled" for op, *_ in failures) - - -def test_emit_success_resets_the_failure_counter() -> None: - from policyengine_observability.destinations.manager import ( - DESTINATION_FAILURE_LIMIT, - ) - - flaky = FailingDestination(fail_first=DESTINATION_FAILURE_LIMIT - 1) - manager, failures = make_manager(destinations=[flaky]) - - for _ in range(DESTINATION_FAILURE_LIMIT + 2): - manager.emit({"event": "x"}, log_type="event", severity="INFO") - - assert flaky in manager.destinations - counts = [ - fields["consecutive_failures"] - for op, _exc, fields in failures - if op == "logging.destination_emit" - ] - assert max(counts) == DESTINATION_FAILURE_LIMIT - 1 - - -def test_sole_disabled_destination_falls_back_to_stdout() -> None: - from policyengine_observability.destinations.manager import ( - DESTINATION_FAILURE_LIMIT, - ) - - flaky = FailingDestination() - manager, _failures = make_manager(destinations=[flaky]) - - for _ in range(DESTINATION_FAILURE_LIMIT + 1): - manager.emit({"event": "x"}, log_type="event", severity="INFO") - - assert flaky not in manager.destinations - assert any( - isinstance(destination, StdoutJsonDestination) - for destination in manager.destinations - ) - - -def test_manager_close_reports_failures_and_closes_the_rest() -> None: - class ExplodingClose(RecordingDestination): - def close(self) -> None: - raise RuntimeError("close failed") - - exploding = ExplodingClose() - closable = ClosableRecordingDestination() - manager, failures = make_manager(destinations=[exploding, closable]) - - manager.close(1.0) # must not raise - - assert closable.closed == 1 - assert any(op == "logging.destination_close" for op, *_ in failures) - - -def test_manager_close_accepts_zero_argument_close() -> None: - """A duck-typed close(self) works under every close path — the - deadline is passed only when the signature accepts it.""" - closable = ClosableRecordingDestination() - manager, failures = make_manager(destinations=[closable]) - - manager.close(1.0) - - assert closable.closed == 1 - assert failures == [] - - -def test_manager_close_shares_one_deadline_across_destinations() -> None: - import time - - deadlines = [] - - class SlowClose(RecordingDestination): - def close(self, deadline_seconds=None) -> None: - deadlines.append(deadline_seconds) - time.sleep(0.05) - - manager, _failures = make_manager(destinations=[SlowClose(), SlowClose()]) - - manager.close(1.0) - - first, second = deadlines - # The second destination only gets what the first one left, so N - # stuck destinations cannot take N times the budget. - assert first <= 1.0 - assert second <= first - 0.04 - - -def test_emit_falls_back_to_stdout_when_configure_crashes( - monkeypatch, -) -> None: - logger = RecordingLogger() - manager, failures = make_manager(loggers={"event": logger}) - - def broken_configure() -> None: - raise RuntimeError("configure exploded") - - monkeypatch.setattr(manager, "configure", broken_configure) - - manager.emit({"event": "x"}, log_type="event", severity="INFO") - - assert manager.configured is True - assert isinstance(manager.destinations[0], StdoutJsonDestination) - assert _emitted_line(logger) == {"event": "x", "severity": "INFO"} - assert any(op == "logging.destination_config" for op, *_ in failures) - - -def test_reconfigure_closes_previous_after_installing_new() -> None: - """Close-phase failure reports route through emit; the replaced - destinations must be closed only after the new ones are installed - so those reports still have a sink.""" - sink_states = [] - manager, _failures = make_manager() - - class CloseProbe(RecordingDestination): - def close(self) -> None: - sink_states.append(list(manager.destinations)) - - manager.destinations = [CloseProbe()] - manager.configured = True - - manager.configure() - - assert len(sink_states) == 1 - assert sink_states[0], "previous destination closed before new install" diff --git a/uv.lock b/uv.lock index 08efba2..70ad15f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,6 @@ version = 1 revision = 3 -requires-python = ">=3.12" +requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.14'", "python_full_version == '3.13.*'", @@ -38,15 +38,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/16/9826f089383c593cdfc4a6e5aca94d9e91ae1692c57af82c3b2aa5e810f7/anyio-4.14.0-py3-none-any.whl", hash = "sha256:dd9b7a2a9799ed6552fde617b2c5df02b7fdd7d88392fc48101e51bae46164d9", size = 123506, upload-time = "2026-06-15T22:00:47.595Z" }, ] -[[package]] -name = "asgiref" -version = "3.11.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/63/40/f03da1264ae8f7cfdbf9146542e5e7e8100a4c66ab48e791df9a03d3f6c0/asgiref-3.11.1.tar.gz", hash = "sha256:5f184dc43b7e763efe848065441eac62229c9f7b0475f41f80e207a114eda4ce", size = 38550, upload-time = "2026-02-03T13:30:14.33Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/0a/a72d10ed65068e115044937873362e6e32fab1b7dce0046aeb224682c989/asgiref-3.11.1-py3-none-any.whl", hash = "sha256:e8667a091e69529631969fd45dc268fa79b99c92c5fcdda727757e52146ec133", size = 24345, upload-time = "2026-02-03T13:30:13.039Z" }, -] - [[package]] name = "blinker" version = "1.9.0" @@ -88,6 +79,19 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/12/4a/3dfd5f7850cbf0d06dc84ba9aa00db766b52ca38d8b86e3a38314d52498c/cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", size = 184344, upload-time = "2025-09-08T23:22:26.456Z" }, + { url = "https://files.pythonhosted.org/packages/4f/8b/f0e4c441227ba756aafbe78f117485b25bb26b1c059d01f137fa6d14896b/cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", size = 180560, upload-time = "2025-09-08T23:22:28.197Z" }, + { url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613, upload-time = "2025-09-08T23:22:29.475Z" }, + { url = "https://files.pythonhosted.org/packages/b8/56/6033f5e86e8cc9bb629f0077ba71679508bdf54a9a5e112a3c0b91870332/cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", size = 216476, upload-time = "2025-09-08T23:22:31.063Z" }, + { url = "https://files.pythonhosted.org/packages/dc/7f/55fecd70f7ece178db2f26128ec41430d8720f2d12ca97bf8f0a628207d5/cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", size = 203374, upload-time = "2025-09-08T23:22:32.507Z" }, + { url = "https://files.pythonhosted.org/packages/84/ef/a7b77c8bdc0f77adc3b46888f1ad54be8f3b7821697a7b89126e829e676a/cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", size = 202597, upload-time = "2025-09-08T23:22:34.132Z" }, + { url = "https://files.pythonhosted.org/packages/d7/91/500d892b2bf36529a75b77958edfcd5ad8e2ce4064ce2ecfeab2125d72d1/cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", size = 215574, upload-time = "2025-09-08T23:22:35.443Z" }, + { url = "https://files.pythonhosted.org/packages/44/64/58f6255b62b101093d5df22dcb752596066c7e89dd725e0afaed242a61be/cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", size = 218971, upload-time = "2025-09-08T23:22:36.805Z" }, + { url = "https://files.pythonhosted.org/packages/ab/49/fa72cebe2fd8a55fbe14956f9970fe8eb1ac59e5df042f603ef7c8ba0adc/cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", size = 211972, upload-time = "2025-09-08T23:22:38.436Z" }, + { url = "https://files.pythonhosted.org/packages/0b/28/dd0967a76aab36731b6ebfe64dec4e981aff7e0608f60c2d46b46982607d/cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", size = 217078, upload-time = "2025-09-08T23:22:39.776Z" }, + { url = "https://files.pythonhosted.org/packages/2b/c0/015b25184413d7ab0a410775fdb4a50fca20f5589b5dab1dbbfa3baad8ce/cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", size = 172076, upload-time = "2025-09-08T23:22:40.95Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8f/dc5531155e7070361eb1b7e4c1a9d896d0cb21c49f807a6c03fd63fc877e/cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", size = 182820, upload-time = "2025-09-08T23:22:42.463Z" }, + { url = "https://files.pythonhosted.org/packages/95/5c/1b493356429f9aecfd56bc171285a4c4ac8697f76e9bbbbb105e537853a1/cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", size = 177635, upload-time = "2025-09-08T23:22:43.623Z" }, { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, @@ -142,6 +146,22 @@ version = "3.4.7" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/d7/b5b7020a0565c2e9fa8c09f4b5fa6232feb326b8c20081ccded47ea368fd/charset_normalizer-3.4.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7", size = 309705, upload-time = "2026-04-02T09:26:02.191Z" }, + { url = "https://files.pythonhosted.org/packages/5a/53/58c29116c340e5456724ecd2fff4196d236b98f3da97b404bc5e51ac3493/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7", size = 206419, upload-time = "2026-04-02T09:26:03.583Z" }, + { url = "https://files.pythonhosted.org/packages/b2/02/e8146dc6591a37a00e5144c63f29fb7c97a734ea8a111190783c0e60ab63/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e", size = 227901, upload-time = "2026-04-02T09:26:04.738Z" }, + { url = "https://files.pythonhosted.org/packages/fb/73/77486c4cd58f1267bf17db420e930c9afa1b3be3fe8c8b8ebbebc9624359/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c", size = 222742, upload-time = "2026-04-02T09:26:06.36Z" }, + { url = "https://files.pythonhosted.org/packages/a1/fa/f74eb381a7d94ded44739e9d94de18dc5edc9c17fb8c11f0a6890696c0a9/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df", size = 214061, upload-time = "2026-04-02T09:26:08.347Z" }, + { url = "https://files.pythonhosted.org/packages/dc/92/42bd3cefcf7687253fb86694b45f37b733c97f59af3724f356fa92b8c344/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265", size = 199239, upload-time = "2026-04-02T09:26:09.823Z" }, + { url = "https://files.pythonhosted.org/packages/4c/3d/069e7184e2aa3b3cddc700e3dd267413dc259854adc3380421c805c6a17d/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4", size = 210173, upload-time = "2026-04-02T09:26:10.953Z" }, + { url = "https://files.pythonhosted.org/packages/62/51/9d56feb5f2e7074c46f93e0ebdbe61f0848ee246e2f0d89f8e20b89ebb8f/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e", size = 209841, upload-time = "2026-04-02T09:26:12.142Z" }, + { url = "https://files.pythonhosted.org/packages/d2/59/893d8f99cc4c837dda1fe2f1139079703deb9f321aabcb032355de13b6c7/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38", size = 200304, upload-time = "2026-04-02T09:26:13.711Z" }, + { url = "https://files.pythonhosted.org/packages/7d/1d/ee6f3be3464247578d1ed5c46de545ccc3d3ff933695395c402c21fa6b77/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c", size = 229455, upload-time = "2026-04-02T09:26:14.941Z" }, + { url = "https://files.pythonhosted.org/packages/54/bb/8fb0a946296ea96a488928bdce8ef99023998c48e4713af533e9bb98ef07/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b", size = 210036, upload-time = "2026-04-02T09:26:16.478Z" }, + { url = "https://files.pythonhosted.org/packages/9a/bc/015b2387f913749f82afd4fcba07846d05b6d784dd16123cb66860e0237d/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c", size = 224739, upload-time = "2026-04-02T09:26:17.751Z" }, + { url = "https://files.pythonhosted.org/packages/17/ab/63133691f56baae417493cba6b7c641571a2130eb7bceba6773367ab9ec5/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d", size = 216277, upload-time = "2026-04-02T09:26:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/06/6d/3be70e827977f20db77c12a97e6a9f973631a45b8d186c084527e53e77a4/charset_normalizer-3.4.7-cp311-cp311-win32.whl", hash = "sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad", size = 147819, upload-time = "2026-04-02T09:26:20.295Z" }, + { url = "https://files.pythonhosted.org/packages/20/d9/5f67790f06b735d7c7637171bbfd89882ad67201891b7275e51116ed8207/charset_normalizer-3.4.7-cp311-cp311-win_amd64.whl", hash = "sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00", size = 159281, upload-time = "2026-04-02T09:26:21.74Z" }, + { url = "https://files.pythonhosted.org/packages/ca/83/6413f36c5a34afead88ce6f66684d943d91f233d76dd083798f9602b75ae/charset_normalizer-3.4.7-cp311-cp311-win_arm64.whl", hash = "sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1", size = 147843, upload-time = "2026-04-02T09:26:22.901Z" }, { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328, upload-time = "2026-04-02T09:26:24.331Z" }, { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061, upload-time = "2026-04-02T09:26:25.568Z" }, { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031, upload-time = "2026-04-02T09:26:26.865Z" }, @@ -236,6 +256,21 @@ version = "7.14.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/9c/a3/3834a5564fe8f32154cd7032400d3c2f9c565b2a373fa671f2bbdad6f634/coverage-7.14.2.tar.gz", hash = "sha256:7a2da3d81cfe17c18038c6d98e6592aa9147d596d056119b0ee612c3c8bd5230", size = 923982, upload-time = "2026-06-20T14:49:30.885Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/04/d5/d0e511247f84fa88ae7da68403cbd3bf9d2a5fc48f5d6618a6846b275632/coverage-7.14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:909f265c8c41f04c824bf741b2601fdcb56cab4bf56e018996b6494192ba0f58", size = 220352, upload-time = "2026-06-20T14:47:28.61Z" }, + { url = "https://files.pythonhosted.org/packages/03/4a/ecaff6db72e6c1782ca51336e391393f1e9cc6e4412d6c3da8b7d5075adf/coverage-7.14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c8102deaf911938233f760426e6a5e287388521de95111d5c8de26c8a1028924", size = 220855, upload-time = "2026-06-20T14:47:29.972Z" }, + { url = "https://files.pythonhosted.org/packages/34/9a/cf950cd8e8df06ee5941276e69f81647005360421be523d5ca18f658e143/coverage-7.14.2-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:851f49e7bd7d1cdaf328f3133942b252d5e3d3380690131f423cba8e435b87f5", size = 251276, upload-time = "2026-06-20T14:47:31.413Z" }, + { url = "https://files.pythonhosted.org/packages/9d/08/f973be32c9a095e4bb2d3a7bdcb2f9c117e39d4062471ffffae3623f6c51/coverage-7.14.2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:04cb445bed86aaf00aaa97d41a8b6e30f100f21e81c34caaec4efc684cb57768", size = 253189, upload-time = "2026-06-20T14:47:32.727Z" }, + { url = "https://files.pythonhosted.org/packages/96/aa/f3a50952ba553d442d94b793e5dede25d426b02e5e011e9a9dd225c002d3/coverage-7.14.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7471bc920d97c51c37ea8127f13b2adca43c3d78c53313b26a1f428e99d2c254", size = 255299, upload-time = "2026-06-20T14:47:34.019Z" }, + { url = "https://files.pythonhosted.org/packages/e0/29/9a4c491986f4d637ed64961ae56721661fc21b6b767d280848d0c708756a/coverage-7.14.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:da5057e1bb257c967feee8ba67f3ebf379e801c7717f238b3d8c9caf00fc8f93", size = 257255, upload-time = "2026-06-20T14:47:35.397Z" }, + { url = "https://files.pythonhosted.org/packages/dd/61/d2a5b48007f6a212f321c36cf5486feb80505d2d00dfb1163aad2da71197/coverage-7.14.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33c0da852e8a40246cd8e20cf3b2fc17ca52a45e9b5f7983c93db26f5d24b87b", size = 251417, upload-time = "2026-06-20T14:47:36.677Z" }, + { url = "https://files.pythonhosted.org/packages/ea/25/8df66ae25b401d4529e1d0617af20d9695d171ea4ffec4ca9dffc5dc37b7/coverage-7.14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:f48a85bb437fab7782021c40bfee6b15146928b96960d008ace41b6901a0f21d", size = 252991, upload-time = "2026-06-20T14:47:38.027Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7b/16bdc9116dd8bf412a421a7227daa65ad9f12bef0685b13c1bd1c12e6d4c/coverage-7.14.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f44e7579a769a21d5b5e3166916bfe30ee175aaffff750324cbb11be2dbec5ad", size = 251051, upload-time = "2026-06-20T14:47:39.26Z" }, + { url = "https://files.pythonhosted.org/packages/0a/f8/b7dbed84274dcc69ddb9c0fe72ec1260830473e0d6c299dcf087a0567f7c/coverage-7.14.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:78853ca3c6ca2f012daa2b07dbabbb8db0f09d4dbe8ee828d294b3445d3f4cd8", size = 254817, upload-time = "2026-06-20T14:47:40.995Z" }, + { url = "https://files.pythonhosted.org/packages/c6/07/4659e6bed01a25a0effb4952e8e75fd157038fe5f2829b0f69c6811c2033/coverage-7.14.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:c9c2795ee3692097ff226ab806005d36bb9691fca9b35353542b57ea749cc830", size = 250772, upload-time = "2026-06-20T14:47:42.306Z" }, + { url = "https://files.pythonhosted.org/packages/26/f4/45019da4cd6cd1df3042476447449d62a76a201f6b3556aa40ac31bce20b/coverage-7.14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2f5cc48a845d755b6db236f8c29c2b54773eb4c7e4ee2ead43812d73718784b0", size = 251679, upload-time = "2026-06-20T14:47:43.703Z" }, + { url = "https://files.pythonhosted.org/packages/92/e5/76d75fa2ffe0285d3f2608d1bb241fc245cf98fe614d52118427dd6ccdaa/coverage-7.14.2-cp311-cp311-win32.whl", hash = "sha256:9c61cb7eaabcfa609c5bc0f5ff5869d72a2f02f17994e5fba5f971de516f3c82", size = 222445, upload-time = "2026-06-20T14:47:45.137Z" }, + { url = "https://files.pythonhosted.org/packages/57/59/696c64547e5c8b9ed31532e9c7a5f9b6474054da93f8ab07f8baf7365c57/coverage-7.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:e715909b0966d1774d8a26e14e2f4a3ae75909dca526901c6306286b2dcbfbdc", size = 222922, upload-time = "2026-06-20T14:47:46.67Z" }, + { url = "https://files.pythonhosted.org/packages/63/72/646a28100462996c11b98e27d6786cd61f48100d1479804846a3e1e5bf9b/coverage-7.14.2-cp311-cp311-win_arm64.whl", hash = "sha256:9193f7150937a4fd836b10eaa123e15d98e961d1fabac07e60adf2d4785f888a", size = 222468, upload-time = "2026-06-20T14:47:48.119Z" }, { url = "https://files.pythonhosted.org/packages/d0/d9/bdd141aa2c605096a8ef63b8435fd4f5fec78946a3cb7b9145840ec78291/coverage-7.14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:37c94712e533ea06f0b1e4d934811c520b1914ce0e4da3916220717aa7a86bc6", size = 220528, upload-time = "2026-06-20T14:47:49.652Z" }, { url = "https://files.pythonhosted.org/packages/02/97/d24ae7d2afc62c54a36313d4dedb655c9afbba3003f0f7f1ae81e97af31f/coverage-7.14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c050bbc7bba94c77e4ed7438f4fda1babe98ab145691d80aa6f60df934a1468b", size = 220883, upload-time = "2026-06-20T14:47:51.036Z" }, { url = "https://files.pythonhosted.org/packages/f8/0e/d8f00efd3df0d63e6843ebcbade9e4119d60f5376753c9705d84b014c775/coverage-7.14.2-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a7af571767a2ee342a171c16fc1b1a07a0bf511606d381703fb7cf397fe49d46", size = 252395, upload-time = "2026-06-20T14:47:52.627Z" }, @@ -347,6 +382,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6c/a0/db537264e234f7273a73ec020873d6d6b39dfd8a53db78b550ca8320440e/cryptography-49.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e", size = 4834820, upload-time = "2026-06-12T20:01:51.847Z" }, { url = "https://files.pythonhosted.org/packages/93/77/8df9eb486495979bccecd1062e2eaf435250e84437040295b57d09048b0b/cryptography-49.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b", size = 4967968, upload-time = "2026-06-12T20:02:12.524Z" }, { url = "https://files.pythonhosted.org/packages/c2/e6/f60198ea8d9dfa15fff9ed4ca02ce362f6eadd9ba757dcc50634c4257b63/cryptography-49.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001", size = 3785547, upload-time = "2026-06-12T20:02:26.847Z" }, + { url = "https://files.pythonhosted.org/packages/63/d3/4a83af35d65e3fad632c926fad684c193ea4398569ccb0bbbc7fe8f5dc9a/cryptography-49.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc1e275c2f1d97b1a6450b8b0ea3ebfa6e087a611c2b26cb2404d48588abab7b", size = 3993685, upload-time = "2026-06-12T20:02:14.883Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a7/f9dac0ab7f80368c56993a7bf638ef9935f825c91902798481fac0898138/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c83782480a4a9da4d0feb51950131ba32e12e70813848b3343f6e18c28a66838", size = 4676239, upload-time = "2026-06-12T20:02:28.793Z" }, + { url = "https://files.pythonhosted.org/packages/d7/70/2ba3769dd0ae167e2f33dfa9592d45db6ff9a61d62ca1a5b3d1bdd09068f/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b39efa323140595abd3ecca8529d321ae50f55f3aa3ba9cc81ea56a6011953d5", size = 4715584, upload-time = "2026-06-12T20:01:27.495Z" }, + { url = "https://files.pythonhosted.org/packages/94/64/2923570ac1c0bd3a737aa366ac3abbbbde273042308b8cde95e2364a6e6a/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:b47db11c2c3525083296069b98ac5221907455e989ae0c2e3008bde851921615", size = 4675885, upload-time = "2026-06-12T20:01:55.49Z" }, + { url = "https://files.pythonhosted.org/packages/ab/f8/614dc7e051418cfe53d55173c1e24c6b0085e89996fe90508c2fdf769aef/cryptography-49.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:084ef1af862eb07ec46d25f68689f2102a9fc0e05ce7b80f14f5fe51e4eef0f6", size = 4715449, upload-time = "2026-06-12T20:02:05.469Z" }, + { url = "https://files.pythonhosted.org/packages/aa/50/a9caea39ad19c431c1a3f8a31114df65b260cdfe67786b6c7e7c040c4c44/cryptography-49.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6", size = 3783731, upload-time = "2026-06-12T20:02:43.319Z" }, ] [[package]] @@ -520,6 +561,16 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/b0/b5/1ff353970a87eda4c98251e34d2dfd214abd4982dc89119c9252a2a482d2/grpcio-1.81.1.tar.gz", hash = "sha256:6fa10a767143a5e82e8eaab53918af0cd8909a57a27f8cb2288b80a613ac671b", size = 13026582, upload-time = "2026-06-11T12:46:51.673Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/52/ea/1c2fa386b718ff493225e61cfc052ef400b4d6ffc54cbe261026432624b5/grpcio-1.81.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:d71d30f2d92f67d944631c523713934fee37292469e182ebcd2c1dd8a64ce53f", size = 6093112, upload-time = "2026-06-11T12:44:52.131Z" }, + { url = "https://files.pythonhosted.org/packages/2b/18/acf45fa8bd1bc5d7b0c2fd3dc4c209379fbd5bb396b440b68a83342226b7/grpcio-1.81.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b137f4bf3ada9dc44d411478decc6ff09a79ed30b306cd2abaa98408c3588137", size = 12074277, upload-time = "2026-06-11T12:44:55.354Z" }, + { url = "https://files.pythonhosted.org/packages/48/d7/ee86a60699b7db039f772a2c4a7e4facc7138984ff42c0130933a0063884/grpcio-1.81.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a3acb384427816dd5d470f47e62137b87f74da694faa8a50147012cf40df276a", size = 6640348, upload-time = "2026-06-11T12:44:59.223Z" }, + { url = "https://files.pythonhosted.org/packages/26/ee/d2de5e47378ffc207d476c230fea3be4d2601edbce9995f4fe45535d4896/grpcio-1.81.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:f9a0ebbe45c29b5e5866593c12b78bd9035f0f0f0d4bc8361680cd580d99db49", size = 7331842, upload-time = "2026-06-11T12:45:02.001Z" }, + { url = "https://files.pythonhosted.org/packages/23/d6/abeda5c2b896a0b341584fe5ac411bbf72e197a9a374c355fb90965e08d2/grpcio-1.81.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a37165cc80b1a368384b383e63a4c38116a10467ae44c904d2d7468c4470ec2", size = 6842229, upload-time = "2026-06-11T12:45:04.76Z" }, + { url = "https://files.pythonhosted.org/packages/10/1c/1f0da7d590b4aeee006826ba568d0e419ca14b23e18f901a3da3e9fba613/grpcio-1.81.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6282caffb41ec326d4cb67ca9cf53b739d1b2f975a2acb498c7418e9f7d9a416", size = 7446096, upload-time = "2026-06-11T12:45:07.499Z" }, + { url = "https://files.pythonhosted.org/packages/6a/81/5c505d508f7c887aa7982d21443a4126597c80d34b0bcf40f9cec576d7f3/grpcio-1.81.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a35009284d0d3d5c2c9601c164a911b8b4331608d98a9a66d47d97bb2f522b70", size = 8445238, upload-time = "2026-06-11T12:45:10.243Z" }, + { url = "https://files.pythonhosted.org/packages/f7/b2/524847365122ee509ca17bcc4e092198b700e94af7bfd5bb5e6dd9f3ee66/grpcio-1.81.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1b22c80559854b789a01fd89e8929b3798a156c0829b5282a8939f33ad4115ad", size = 7873989, upload-time = "2026-06-11T12:45:13.102Z" }, + { url = "https://files.pythonhosted.org/packages/18/fa/07c037c50b006909d1d13a5848774f8aa7b242f70dc03a035c64eea0e6db/grpcio-1.81.1-cp311-cp311-win32.whl", hash = "sha256:428bec0161b48d8cf583c068591bc0016d0d9cfff52462b72b3884861ea768c5", size = 4202223, upload-time = "2026-06-11T12:45:16.166Z" }, + { url = "https://files.pythonhosted.org/packages/41/ed/6bff15376920942fac6b95b9802752b837437172c9e8fc2d3170546b89cc/grpcio-1.81.1-cp311-cp311-win_amd64.whl", hash = "sha256:30e825f6848d9f18bba350ed6c75c1b02a0b5184474a31db9a32b1fa66fd8c79", size = 4941303, upload-time = "2026-06-11T12:45:18.724Z" }, { url = "https://files.pythonhosted.org/packages/85/07/9a979c81738863a738dc23d65177056e71fbb2db817740ed870b33434e7a/grpcio-1.81.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:8b39472beafc0bdcafc4c8c73ad082ebfdb449d566897a61e7acb4fa88089115", size = 6053264, upload-time = "2026-06-11T12:45:21.017Z" }, { url = "https://files.pythonhosted.org/packages/75/95/539706ca0d3bd40dbad583dc56fd883da941f37556b629132da5762781b9/grpcio-1.81.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:12b7524c88d4026d3dcb7b0ebe16b6714f3b4af402ddd0f0639ab064a00c87c3", size = 12052560, upload-time = "2026-06-11T12:45:23.652Z" }, { url = "https://files.pythonhosted.org/packages/e0/44/f257b7e0bd69c93b06c6cb8ac8d1b901ccb42bedabd83c1a4c77a71f8810/grpcio-1.81.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1e123f9b37edb8375fd74130d1f69c944bbf0a7b06761ae7211154b8759e94d2", size = 6595983, upload-time = "2026-06-11T12:45:26.963Z" }, @@ -648,6 +699,17 @@ version = "3.0.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, + { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, + { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, + { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, + { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, + { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, + { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, @@ -705,6 +767,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "nodeenv" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.43.0" @@ -765,69 +836,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/20/b685ed7af2e17c29ffc8af56f1fa8bc2033258fc30fb0d2b722f49d13ba0/opentelemetry_exporter_otlp_proto_http-1.43.0-py3-none-any.whl", hash = "sha256:647f603aa8efdbdb4dbff842e0729d0406a6fff26b295a72d3d60e7d963b2610", size = 21795, upload-time = "2026-06-24T15:19:43.164Z" }, ] -[[package]] -name = "opentelemetry-instrumentation" -version = "0.64b0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "packaging" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7e/97/02fe6e1c8b1ffac42d0b429c18080edb24e0e0d18c86612edf72b5752382/opentelemetry_instrumentation-0.64b0.tar.gz", hash = "sha256:b47d528dead6271d7743114417eb67fc915bd9258111c48dbf9a4951d2efa88d", size = 41935, upload-time = "2026-06-24T15:19:12.951Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/0c/cb9fe342de5299c7af24582eb7d788661cc53a1c4b904da92309caaa9417/opentelemetry_instrumentation-0.64b0-py3-none-any.whl", hash = "sha256:133ab7ffca796557aec059bf6be3190a34b6dea987f25be3d9409e230cbdad8b", size = 35880, upload-time = "2026-06-24T15:18:17.277Z" }, -] - -[[package]] -name = "opentelemetry-instrumentation-asgi" -version = "0.64b0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "asgiref" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-instrumentation" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "opentelemetry-util-http" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/85/0c/71c696fccb86d37af383ea1604af4729fabad0af2fabaf203e4c79c1e859/opentelemetry_instrumentation_asgi-0.64b0.tar.gz", hash = "sha256:4dd3eee566a4303f8e6b9b84f2a0a7abc57a6640df768926c68a3868bf5b2090", size = 26136, upload-time = "2026-06-24T15:19:17.003Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/20/218b65a63d847a7ed28d1bea84c39234689160b74480b8702272e37f4240/opentelemetry_instrumentation_asgi-0.64b0-py3-none-any.whl", hash = "sha256:e0840b66e15303a9254b0540946010bd008aa0504f4d89b8e1b7fb63490a36f0", size = 15906, upload-time = "2026-06-24T15:18:23.107Z" }, -] - -[[package]] -name = "opentelemetry-instrumentation-fastapi" -version = "0.64b0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-instrumentation" }, - { name = "opentelemetry-instrumentation-asgi" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "opentelemetry-util-http" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/db/a1/52282e2cc5c08f4df13b087896d9907258fe2ff4f34035d3b21aa92684c3/opentelemetry_instrumentation_fastapi-0.64b0.tar.gz", hash = "sha256:05f75149929e433c1630de381688e650bf651c1e1cce7f9a7b649a807dac8a98", size = 26236, upload-time = "2026-06-24T15:19:27.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/72/2d/4f48dc2d6f289f2febc5b871940dbea9f3b04ab9186d23342581b49cb984/opentelemetry_instrumentation_fastapi-0.64b0-py3-none-any.whl", hash = "sha256:43cbbfb2d3079dc81104478a2950ae93ac6d0e90a5020fa3987a236f8f2bdef1", size = 13263, upload-time = "2026-06-24T15:18:38.553Z" }, -] - -[[package]] -name = "opentelemetry-instrumentation-httpx" -version = "0.64b0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-instrumentation" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "opentelemetry-util-http" }, - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0d/2a/2893a8781b93894f1e8014904c0342da7e4302de6597c2d5c0cb6c1a552e/opentelemetry_instrumentation_httpx-0.64b0.tar.gz", hash = "sha256:c2cfcd03d3665762860ebd0c28038c6e47fbb48d7942dec31dd75fc634d25c92", size = 23555, upload-time = "2026-06-24T15:19:29.107Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/29/a20309bd3f5a8051b61ca475e78623410c3b077e3deccae19aa4f8b5b9a2/opentelemetry_instrumentation_httpx-0.64b0-py3-none-any.whl", hash = "sha256:04829e5723941b5ceb0c88b44d63983e226b5c75b2b2e34a57739fdd0e060608", size = 16336, upload-time = "2026-06-24T15:18:41.412Z" }, -] - [[package]] name = "opentelemetry-proto" version = "1.43.0" @@ -867,15 +875,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/ca/23ba87a221b574a7c5a99d48849d80bfe8b047624681357e2b002e566187/opentelemetry_semantic_conventions-0.64b0-py3-none-any.whl", hash = "sha256:ea77e85e354b8f604ddbe5f3d9135216f982fa4d77e5859ac30f6d8a50505aa6", size = 203713, upload-time = "2026-06-24T15:19:53.339Z" }, ] -[[package]] -name = "opentelemetry-util-http" -version = "0.64b0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/1b/1029a805fd7242f7dfce91633b244c3b14a94d703232878f71e01ce862b1/opentelemetry_util_http-0.64b0.tar.gz", hash = "sha256:8a86a220dbfc56d736f47f1e5c4e7932a21fcf69052312e1bcf166444dc79322", size = 11102, upload-time = "2026-06-24T15:19:48.974Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1c/c7/5f8ec5b30546f2dc22cd5fc5759bce2ab5be6e89a2e710a405ac9ef64ed3/opentelemetry_util_http-0.64b0-py3-none-any.whl", hash = "sha256:c1e5350d25507c1afcd6076cf9ac062485a0a4f79cd9971366996fd3056bacdb", size = 8204, upload-time = "2026-06-24T15:19:09.02Z" }, -] - [[package]] name = "packaging" version = "26.2" @@ -896,27 +895,25 @@ wheels = [ [[package]] name = "policyengine-observability" -version = "1.4.1" +version = "2.0.0" source = { editable = "." } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-grpc" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, - { name = "opentelemetry-instrumentation-fastapi" }, - { name = "opentelemetry-instrumentation-httpx" }, - { name = "opentelemetry-sdk" }, -] [package.optional-dependencies] all = [ { name = "fastapi" }, { name = "flask" }, + { name = "google-auth" }, { name = "google-cloud-logging" }, { name = "httpx" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, ] dev = [ { name = "build" }, { name = "coverage" }, + { name = "pyright" }, { name = "pytest" }, { name = "ruff" }, { name = "towncrier" }, @@ -928,11 +925,26 @@ flask = [ { name = "flask" }, ] google = [ + { name = "google-auth" }, { name = "google-cloud-logging" }, ] httpx = [ { name = "httpx" }, ] +otel = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, +] +otlp-grpc = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-sdk" }, +] +otlp-http = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, +] [package.metadata] requires-dist = [ @@ -942,16 +954,25 @@ requires-dist = [ { name = "fastapi", marker = "extra == 'fastapi'" }, { name = "flask", marker = "extra == 'all'", specifier = ">=2.2" }, { name = "flask", marker = "extra == 'flask'", specifier = ">=2.2" }, + { name = "google-auth", marker = "extra == 'all'", specifier = ">=2.38.0" }, + { name = "google-auth", marker = "extra == 'google'", specifier = ">=2.38.0" }, { name = "google-cloud-logging", marker = "extra == 'all'", specifier = ">=3.15.0" }, { name = "google-cloud-logging", marker = "extra == 'google'", specifier = ">=3.15.0" }, { name = "httpx", marker = "extra == 'all'" }, { name = "httpx", marker = "extra == 'httpx'" }, - { name = "opentelemetry-api", specifier = ">=1.43.0" }, - { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.43.0" }, - { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.43.0" }, - { name = "opentelemetry-instrumentation-fastapi", specifier = ">=0.64b0" }, - { name = "opentelemetry-instrumentation-httpx", specifier = ">=0.64b0" }, - { name = "opentelemetry-sdk", specifier = ">=1.43.0" }, + { name = "opentelemetry-api", marker = "extra == 'all'", specifier = ">=1.43.0" }, + { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.43.0" }, + { name = "opentelemetry-api", marker = "extra == 'otlp-grpc'", specifier = ">=1.43.0" }, + { name = "opentelemetry-api", marker = "extra == 'otlp-http'", specifier = ">=1.43.0" }, + { name = "opentelemetry-exporter-otlp-proto-grpc", marker = "extra == 'all'", specifier = ">=1.43.0" }, + { name = "opentelemetry-exporter-otlp-proto-grpc", marker = "extra == 'otlp-grpc'", specifier = ">=1.43.0" }, + { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'all'", specifier = ">=1.43.0" }, + { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otlp-http'", specifier = ">=1.43.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'all'", specifier = ">=1.43.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.43.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'otlp-grpc'", specifier = ">=1.43.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'otlp-http'", specifier = ">=1.43.0" }, + { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.405" }, { name = "pytest", marker = "extra == 'dev'" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.9.0" }, { name = "towncrier", marker = "extra == 'dev'", specifier = ">=24.8.0" }, @@ -1039,6 +1060,21 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, @@ -1099,10 +1135,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, ] [[package]] @@ -1123,6 +1171,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bd/24/12818598c362d7f300f18e74db45963dbcb85150324092410c8b49405e42/pyproject_hooks-1.2.0-py3-none-any.whl", hash = "sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913", size = 10216, upload-time = "2024-09-29T09:24:11.978Z" }, ] +[[package]] +name = "pyright" +version = "1.1.414" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nodeenv" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/1b/244c7b710031ada80f27e579ec20d28a2285dfc318fed0339866b1047f12/pyright-1.1.414.tar.gz", hash = "sha256:523c0a97c60da6333234955c277730c9cf4f5bd6d5399e7b7d2b0fc5d3599524", size = 4154638, upload-time = "2026-09-10T12:26:53.181Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/ba/18b6e682ead424ad24bcc134339ae5d1b931cd9ae260540592a058a91279/pyright-1.1.414-py3-none-any.whl", hash = "sha256:2a6b4b3298c9eec174c5ed83bd338de6eee82df2992f3e1930e6199d381be36f", size = 6225049, upload-time = "2026-09-10T12:26:51.427Z" }, +] + [[package]] name = "pytest" version = "9.1.1" @@ -1246,67 +1307,3 @@ sdist = { url = "https://files.pythonhosted.org/packages/dd/b2/381be8cfdee792dd1 wheels = [ { url = "https://files.pythonhosted.org/packages/93/8c/2e650f2afeb7ee576912636c23ddb621c91ac6a98e66dc8d29c3c69446e1/werkzeug-3.1.8-py3-none-any.whl", hash = "sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50", size = 226459, upload-time = "2026-04-02T18:49:12.72Z" }, ] - -[[package]] -name = "wrapt" -version = "2.2.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fe/a4/282c8e64300a59fc834518a54bf0afabb4ff9218b5fa76958b450459a844/wrapt-2.2.2.tar.gz", hash = "sha256:0788e321027c999bf221b667bd4a54aaefd1a36283749a860ac3eb77daed0302", size = 129068, upload-time = "2026-06-20T23:49:44.49Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/85/180b40628b23772692a0c76e8030114e1c0ae068470ed531919f0a5f2a4a/wrapt-2.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8417fd3c674d3c8023d080292d29301531a12daf8bd938dd419710dd2f464f2b", size = 81484, upload-time = "2026-06-20T23:47:59.924Z" }, - { url = "https://files.pythonhosted.org/packages/94/f2/21c90f2a16689702e2aaff45795b11018dff2c9b1242bac10d225483f676/wrapt-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e7070c7472582e31af3dfc2622b2381a0df7435110a9388ed8db5ffbce67efb", size = 82151, upload-time = "2026-06-20T23:48:01.303Z" }, - { url = "https://files.pythonhosted.org/packages/5f/b3/7e6e9fcf4fe7e1b69a49fe6cc5a44e8224bab6283c5233c97e132f14908e/wrapt-2.2.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e096c9d39a59b35b63c9aacfbbbec2088ff51ff1fc31051acc60a07f42f273a", size = 169828, upload-time = "2026-06-20T23:48:02.719Z" }, - { url = "https://files.pythonhosted.org/packages/0b/43/894f132d857ed5a9904d937baf368badcbe5ea9e436e2f1930fe21c9f1f0/wrapt-2.2.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d1a6050405bf334be33bf66296f113563622972a34900ae6fa60fd283a1a900", size = 171544, upload-time = "2026-06-20T23:48:04.266Z" }, - { url = "https://files.pythonhosted.org/packages/29/de/3c833e03725b477e9ea34028224dd21a48781830101e4e036f77e8b6b102/wrapt-2.2.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10adb01371408c6de504a6658b9886480f1a4919a83752748a387a504a21df79", size = 160663, upload-time = "2026-06-20T23:48:05.708Z" }, - { url = "https://files.pythonhosted.org/packages/33/be/27edce350b24e3054d9d047f65f16d4c4d4c1f3f31c4278a1f8a95c723c8/wrapt-2.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3442eee2a5798f9b451f1b2cd7518ce8b7e28a2a364696c414460a0e295c012a", size = 169387, upload-time = "2026-06-20T23:48:07.243Z" }, - { url = "https://files.pythonhosted.org/packages/e2/c4/9fd9679af8bf38e146652c7f47b6b352c3e5795b4ad1c0b7f94e15ac2aa7/wrapt-2.2.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6c99012a22f735a85eed7c4b86a3e99c30fdd57d9e115b2b45f796264b58d0bf", size = 158849, upload-time = "2026-06-20T23:48:08.91Z" }, - { url = "https://files.pythonhosted.org/packages/bc/c2/aa6c0c2206803068c6859dabe01f8c84c43744da93d4c67b8946d21655ee/wrapt-2.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b686cfc008776a3952d6213cb296ed7f45d782a8453936406faa89eac0835ab", size = 168147, upload-time = "2026-06-20T23:48:10.374Z" }, - { url = "https://files.pythonhosted.org/packages/42/63/3eb25da41049d20ae18fcab2dd8b056e02387c4bfa626cbdfb7c3b872e4f/wrapt-2.2.2-cp312-cp312-win32.whl", hash = "sha256:ef2cce266b5b0b07e19fa82e59673b81142b7a3607c8ed1254113d048ed668da", size = 77734, upload-time = "2026-06-20T23:48:11.769Z" }, - { url = "https://files.pythonhosted.org/packages/da/09/0390e008a305360948fa9ce69507d041ac12cb2ee5d28e34467e2ee79391/wrapt-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:abf8c20a2d72ee69e16328b3c91342c446e723bfe48bfcc4dded3b9722ac027f", size = 80585, upload-time = "2026-06-20T23:48:13.117Z" }, - { url = "https://files.pythonhosted.org/packages/d3/b3/84c445c66969f2d3457276b183a48c91097d59bbef9af6c075366b0f8c36/wrapt-2.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:c6c64c5d02578bc4c4bca4f0aef1504de933c1d5b4ac2710b9131111459506c8", size = 79553, upload-time = "2026-06-20T23:48:14.5Z" }, - { url = "https://files.pythonhosted.org/packages/43/fc/f32f4b22c6511173c11d9e541ab4e7d8467a0f1b3455acaf784115d31ff8/wrapt-2.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e8b648270c613720a202d9a45ebabc33261b22c3a839b115ac5bce8c0bb0d69", size = 81296, upload-time = "2026-06-20T23:48:15.881Z" }, - { url = "https://files.pythonhosted.org/packages/72/06/4d117d5d77a9344776c0248b24dae3d3dd2f58e5f765fa08cf887072e719/wrapt-2.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6fb7e94e8fe3e4c3067bb1653a91cce7c5e83acc119fdd41501b1bf74654617", size = 81841, upload-time = "2026-06-20T23:48:17.262Z" }, - { url = "https://files.pythonhosted.org/packages/15/ff/63ad96f98eb58a742b1a20d80f21da88924405910149950b912368150468/wrapt-2.2.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb18fc51e813df0d9c98049e3bf2298a5495a648602040e21fa3c7329371159e", size = 167882, upload-time = "2026-06-20T23:48:18.764Z" }, - { url = "https://files.pythonhosted.org/packages/20/1f/8bb62d8933df7acf3247194e6e9fc68edf9d2fa203252c89c94b319dd472/wrapt-2.2.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94b00b00f806eb3ef2abe9049ed45994a81ee9284884d96e6b8314927c6cea3d", size = 167411, upload-time = "2026-06-20T23:48:20.315Z" }, - { url = "https://files.pythonhosted.org/packages/17/09/8789dcb09ee1de715727db7521aabbb68ffa68dfade3a49468440cfced49/wrapt-2.2.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:62415fd095bc590b842b6d092f2b5d9ccbaeb7e0b28535c03dcea2718b48636b", size = 158607, upload-time = "2026-06-20T23:48:21.728Z" }, - { url = "https://files.pythonhosted.org/packages/9c/20/66e02562d53ee67d841f175e38e3c993c2d78a3e104c576cad61c028b43c/wrapt-2.2.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a41e758d80dc0ab8c210f641ac892009d356cf1f955d97db544c8dd317b4d14c", size = 166367, upload-time = "2026-06-20T23:48:23.177Z" }, - { url = "https://files.pythonhosted.org/packages/bd/a3/832ac4e41222fb263b3042d42c2f08d305db7d0f0c9b1d3a271a9eede8f6/wrapt-2.2.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:b84cd4058001c9727b0e9980b7a9e66325b5ca748b1b578e822cade1bc6b304f", size = 157176, upload-time = "2026-06-20T23:48:24.711Z" }, - { url = "https://files.pythonhosted.org/packages/b7/01/1bd5e4d2df9c0178989ac8da9186543465388588ee2ef153e2591accebef/wrapt-2.2.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:26fc73a1b15e0946d2942b9a4426d162b51676338327dc067ccd8d2d76385f94", size = 167025, upload-time = "2026-06-20T23:48:26.118Z" }, - { url = "https://files.pythonhosted.org/packages/1c/69/583ed25291ab53e1ec117135fb1c33425e2f46d2bc8f29c17f7a94cf4274/wrapt-2.2.2-cp313-cp313-win32.whl", hash = "sha256:3c4095803491f6ef72128914c28ec05bbad9758433bb35f6715a3e9c8e46fb2d", size = 77605, upload-time = "2026-06-20T23:48:27.643Z" }, - { url = "https://files.pythonhosted.org/packages/29/68/e69fc6d06e1523c68e0d00f95c9aed1158ce9908ee41603f7f2eae3d5db6/wrapt-2.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:2cb07f414fab25dbe6b5c7398e1491423a5c81a6209533639969a6c928d474a4", size = 80508, upload-time = "2026-06-20T23:48:29.013Z" }, - { url = "https://files.pythonhosted.org/packages/55/21/fe7a393d9e5dc0923bed8f5d857e9dcff210f1fa0888c02cc8f3ffaa55aa/wrapt-2.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:1fc7691f070220215cccb2a20836b9adbaecb8ff22ad47abe63de5f110994fac", size = 79565, upload-time = "2026-06-20T23:48:30.429Z" }, - { url = "https://files.pythonhosted.org/packages/b6/e5/c120d13bf5091164f68c3c1657e84f16f57e71d978421b626393ac5bd7eb/wrapt-2.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ec8f83949028366531383603139403cac7a826e4011955813cdd640017845ce5", size = 83264, upload-time = "2026-06-20T23:48:31.807Z" }, - { url = "https://files.pythonhosted.org/packages/d3/b0/d4a1eb97e0e286625bdf21bc7f702637f9607787ffbbdb5ec14d50c79dbf/wrapt-2.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4b481fb0c40d9fd90a5809911208da700987d373a20a4709dc9e3944af7a6bec", size = 83791, upload-time = "2026-06-20T23:48:33.482Z" }, - { url = "https://files.pythonhosted.org/packages/18/1e/f060df47755e87b57684cee7bfc1362b204df55fac96ffebc0631b697b79/wrapt-2.2.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0065a3b657cec06813b4241d2462ccec287f6863103d7445b725fb3a889736f9", size = 203399, upload-time = "2026-06-20T23:48:34.97Z" }, - { url = "https://files.pythonhosted.org/packages/c4/de/2316a757a1abb6453700b79d83e532146dcef2611348282d4d8889792161/wrapt-2.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30f7424af5c5c345b7f26490e097f74a2ef45b3d08b664dc33571aee3bd3b56c", size = 210461, upload-time = "2026-06-20T23:48:36.569Z" }, - { url = "https://files.pythonhosted.org/packages/ed/29/d1160785ae18ca2495a6d82a21154103d74f656c9fd457fb35f6b11b965a/wrapt-2.2.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:07fdcb012821859168641acf68afad61ef9783cf37100af85f152550e9677194", size = 195313, upload-time = "2026-06-20T23:48:38.175Z" }, - { url = "https://files.pythonhosted.org/packages/f5/2d/7caa9598ae61a9cf0989cc501739cbeeb7d650ab3193cca1407b9af0c6ab/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f90038ab58fafb584801ca62d72384d7d5225d93c76f7b773c22fae545bd8066", size = 206116, upload-time = "2026-06-20T23:48:39.804Z" }, - { url = "https://files.pythonhosted.org/packages/ac/02/281ea1088b8650d865f311b35cf86fd21df89128e2909714f1161e01c9d0/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:c5d7825491bfa2d08b97e9557768987952c7b9ae687d06c3320b40a37ccb7f20", size = 192668, upload-time = "2026-06-20T23:48:41.346Z" }, - { url = "https://files.pythonhosted.org/packages/be/7d/976e2d5b4b5c5babda40974edd54d0a5585cb60132ed86b46f4b80239b16/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ad520e6daa9bbf136f14de735474dbec7dcc0891f718e1d274ce8dc92e645af", size = 198891, upload-time = "2026-06-20T23:48:43.056Z" }, - { url = "https://files.pythonhosted.org/packages/59/b7/e47651797c097f75a37e2ce86dcf04048ff576f3a674f7c558df7b5e9622/wrapt-2.2.2-cp313-cp313t-win32.whl", hash = "sha256:25904acb9475f46c24fe0423dbc8fda8cc5fbc282ab3dc6e72e919748c53f4e9", size = 78537, upload-time = "2026-06-20T23:48:44.509Z" }, - { url = "https://files.pythonhosted.org/packages/d1/6f/9fa5d59fb06d890defb5a8f727ce6a14d2932c8760153f96956628559fee/wrapt-2.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:305d4c247d61c4115794a169141823c62f719525ddb90b23aa332741c77d2c28", size = 82005, upload-time = "2026-06-20T23:48:46.391Z" }, - { url = "https://files.pythonhosted.org/packages/15/80/4c7bd9873d1f9f7d138d93556b500469dbe24f42710b877519c2b9eb380d/wrapt-2.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:c20279cd1a29800815d7b2d6338b60a6c6e78263f9d6e62e0eda251ba9cae2d0", size = 80762, upload-time = "2026-06-20T23:48:47.964Z" }, - { url = "https://files.pythonhosted.org/packages/24/05/7fd9c3f83b2c74cbfc572a0b88aa37431e04bd8aed70d2c0efd3464206de/wrapt-2.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0e64826f920c42d9d9f87e8cc09ffae66c51ede12d59061a5a426deb9aa71745", size = 81341, upload-time = "2026-06-20T23:48:49.39Z" }, - { url = "https://files.pythonhosted.org/packages/4b/68/1bfa43100dd90d4ef74a05897b86275cf57e1313ca14aae2545bc9f872c9/wrapt-2.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dcaa5e1451bd8751d7bd1568dfa3321c78092a52a7ecb5d1a0f18a5791e1fd00", size = 81921, upload-time = "2026-06-20T23:48:50.986Z" }, - { url = "https://files.pythonhosted.org/packages/74/eb/df7b7f0b631dbbc750f39be27d8b55f65777d8ac86da80e12be41a644c4b/wrapt-2.2.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0abfd648dac9ac9c5b3aa9b523d27f1789046640b58dcd5652a720ddb325e1fc", size = 167713, upload-time = "2026-06-20T23:48:52.598Z" }, - { url = "https://files.pythonhosted.org/packages/4d/9a/d1bd36f6d088c8e652a9383cabbd49af30b8c576302a7eccddbab6963e3f/wrapt-2.2.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f4bfd8d1eb438153eff8b8cfe87f032ba65731e1ce06138b5090f745a33f6f95", size = 166779, upload-time = "2026-06-20T23:48:54.33Z" }, - { url = "https://files.pythonhosted.org/packages/4c/ae/24ffacd4187fac2740a1972093929e836dea092d42c87d728cd98fee11a6/wrapt-2.2.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c427c9d06d859848a69f0d928fe28b5c33a941b2265d10a0e1f15cd244f1ee33", size = 158407, upload-time = "2026-06-20T23:48:55.944Z" }, - { url = "https://files.pythonhosted.org/packages/a3/ed/974427668249a356051e8d67d47fa54ef6c777f0fcf3bae9d292c047d4b6/wrapt-2.2.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4250b43d1a129d947e083c4dc6baf333c9bb34edd26f912d5b0457841fc858ab", size = 166594, upload-time = "2026-06-20T23:48:57.617Z" }, - { url = "https://files.pythonhosted.org/packages/fb/5f/e1d7c6e4523f78db2fbd7826babd0348da1d5e0834c4f918b9ab5757dfae/wrapt-2.2.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:173e5bb5ca350a6e0abab60b7ec7cdd7992a814cb14b4de670a28f067f105663", size = 157068, upload-time = "2026-06-20T23:48:59.171Z" }, - { url = "https://files.pythonhosted.org/packages/1e/c1/7ebd1027f00700c0b0233b20aceef2b4784294ed64971424c4a78e069e34/wrapt-2.2.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aa14b01804bce36c6d63d7b6a4f55df390f29f8648cc13a1f40b166f4d54680d", size = 166470, upload-time = "2026-06-20T23:49:00.737Z" }, - { url = "https://files.pythonhosted.org/packages/99/eb/974e471a6a978b8180186b8a9dc5ae3361ce269a967190b709b8ce17abfb/wrapt-2.2.2-cp314-cp314-win32.whl", hash = "sha256:58f9f8d637c9a6e245c6ef5b109b67ec187d2faed23d1405656b51d96e0a5b56", size = 78062, upload-time = "2026-06-20T23:49:02.327Z" }, - { url = "https://files.pythonhosted.org/packages/49/ec/e1281156cdc7a66693838ad7a0865ad641c74abd337a957d668b575aaffb/wrapt-2.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:385cb1866f20479e83299af585375bfa0a4b0c6c9907a981483ea782ea8ae406", size = 80832, upload-time = "2026-06-20T23:49:03.837Z" }, - { url = "https://files.pythonhosted.org/packages/45/7d/1b6b5ddd94005a2dac97a4490c9838f3154977850d633abcb65b30089437/wrapt-2.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:8ffbeaea6771a6eba6e6eeb09767864995726bc8240bb54baf88a9bb1db34d5c", size = 80029, upload-time = "2026-06-20T23:49:05.237Z" }, - { url = "https://files.pythonhosted.org/packages/b0/33/9ebcf8aafe91c601127cbd93708c16aa8f688f34a10bf004046803ecdc4f/wrapt-2.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:09f811d43f6f33ec7515f0be76b159569f4057ab54d3e079c3204dddb90afa2a", size = 83357, upload-time = "2026-06-20T23:49:06.632Z" }, - { url = "https://files.pythonhosted.org/packages/39/38/ec45b635153327b52e52732a0ea980e5f00b7efba65f9e018828f1e69daa/wrapt-2.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a795d3c06e5fbf9ea2f13196180b77aeab1b4685917256ee0d014cc163d90063", size = 83794, upload-time = "2026-06-20T23:49:08.098Z" }, - { url = "https://files.pythonhosted.org/packages/4e/ea/1a89e6d3b7a83c3affe5c09cde77792c947e63e4bc85ad84cd5bb9abb0d8/wrapt-2.2.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:45c2f2768e790c9f8db90f239ef23a2af8e7570f25a35619ef902df4a738447f", size = 203362, upload-time = "2026-06-20T23:49:09.811Z" }, - { url = "https://files.pythonhosted.org/packages/19/d8/3b58763d9863b5a73771c0d97110f9595d248db454009e07e1535ee905a4/wrapt-2.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbf00ee0cb55ec24e2b0995a71942b85b21a066db8f3f46e1dbfdb9433ffba81", size = 210449, upload-time = "2026-06-20T23:49:11.521Z" }, - { url = "https://files.pythonhosted.org/packages/2d/6f/17fd9e053103d8be148d20d5d7505facc72d5fe1f9127973904ceaed79cf/wrapt-2.2.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2252f77663651b89255895f58cc6ac08fcb206d4371813e5af61bb62d4f7689c", size = 195349, upload-time = "2026-06-20T23:49:13.346Z" }, - { url = "https://files.pythonhosted.org/packages/ef/04/d0d1ccaaa12cb7dccf28a23f0279a608ba498f71e81d949d5ed54bcfd5c1/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cd7181ab1c31192ff5219269830744b5a62020b3a6d433588c4f1c95b8f8bff", size = 206099, upload-time = "2026-06-20T23:49:15.051Z" }, - { url = "https://files.pythonhosted.org/packages/44/b3/e8aa07b619890a2aa6cde1931b1887abb08820721b564a5f80b7ca3f3aa0/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:6fe35fd51b74867d8b80174c277bd6bbf6a73e443f908129dc531c4b688a20d5", size = 192728, upload-time = "2026-06-20T23:49:16.854Z" }, - { url = "https://files.pythonhosted.org/packages/b7/f0/1819fb50f0d3c9bd758d8a83b56f1b470dee8b5b8eac8702b7c137cea9d4/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:11d95fc2fbad3163596c39d440e6f21ca9fccece74b56e30a37ac2fca786a07c", size = 198842, upload-time = "2026-06-20T23:49:18.504Z" }, - { url = "https://files.pythonhosted.org/packages/67/7c/e88313f16a99930b899ef970d91c281544a470749a359decad994483bbda/wrapt-2.2.2-cp314-cp314t-win32.whl", hash = "sha256:d8a15813215f33fa83667bfc978b300e35669ea8bb424e970a1426bcb7bc6cca", size = 79059, upload-time = "2026-06-20T23:49:20.107Z" }, - { url = "https://files.pythonhosted.org/packages/a0/4f/ac12fda57a55068a094ec42851fb0a40e8489d8941863d517452de62e507/wrapt-2.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d09db0f7e8357060d3c38fc22a018aba683a796bf184360fd1a58f6fc180dc77", size = 82462, upload-time = "2026-06-20T23:49:21.631Z" }, - { url = "https://files.pythonhosted.org/packages/48/a7/df732dac86d9b2027c56bd163dbc883e037b16c3469614752e148d219c61/wrapt-2.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:f32fe639c39561ccc187bcae17e9271be0eb45f1c2952510d2f29b33ab577347", size = 81182, upload-time = "2026-06-20T23:49:23.199Z" }, - { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" }, -]