Configure and instrument simulation observability

.github/scripts/deploy-cloud-run-simulation-entry.sh

@@ -77,6 +77,10 @@ required_variables=(
   OLD_GATEWAY_AUTH_AUDIENCE_VALUE
   OLD_GATEWAY_AUTH_CLIENT_ID_VALUE
   OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME
+  OBSERVABILITY_SERVICE_NAMESPACE
+  OBSERVABILITY_TRACE_PROJECT_ID
+  OTEL_EXPORTER_OTLP_ENDPOINT
+  POLICYENGINE_OTEL_GOOGLE_AUDIENCE
   RUNNER_TEMP
 )
 for variable_name in "${required_variables[@]}"; do
@@ -148,6 +152,14 @@ jq -n '
     OLD_GATEWAY_AUTH_ISSUER: env.OLD_GATEWAY_AUTH_ISSUER_VALUE,
     OLD_GATEWAY_AUTH_AUDIENCE: env.OLD_GATEWAY_AUTH_AUDIENCE_VALUE,
     OLD_GATEWAY_AUTH_CLIENT_ID: env.OLD_GATEWAY_AUTH_CLIENT_ID_VALUE,
+    OBSERVABILITY_SERVICE_NAMESPACE: env.OBSERVABILITY_SERVICE_NAMESPACE,
+    OBSERVABILITY_TRACE_PROJECT_ID: env.OBSERVABILITY_TRACE_PROJECT_ID,
+    OTEL_EXPORTER_OTLP_ENDPOINT: env.OTEL_EXPORTER_OTLP_ENDPOINT,
+    OTEL_EXPORTER_OTLP_PROTOCOL: "grpc",
+    OTEL_TRACES_EXPORTER: "otlp",
+    OTEL_METRICS_EXPORTER: "otlp",
+    OTEL_TRACES_SAMPLER_ARG: "1.0",
+    POLICYENGINE_OTEL_GOOGLE_AUDIENCE: env.POLICYENGINE_OTEL_GOOGLE_AUDIENCE,
     STAGE12_ENABLED: (
       if env.DEPLOY_STAGE12_V2 == "true" then env.STAGE12_ENABLED_VALUE else "0" end
     )

.github/scripts/modal-sync-secrets.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # Sync secrets from GitHub to Modal environment
 # Usage: ./modal-sync-secrets.sh <modal-environment> <gh-environment>
-# Required env vars: LOGFIRE_TOKEN, HF_TOKEN
+# Required env vars: HF_TOKEN
 # Optional env vars: GCP_CREDENTIALS_JSON
 
 set -euo pipefail
@@ -54,13 +54,6 @@ if truthy "${GATEWAY_AUTH_REQUIRED:-}" && [ ${#missing[@]} -gt 0 ]; then
   exit 1
 fi
 
-# Sync Logfire secret
-uv run modal secret create policyengine-logfire \
-  "LOGFIRE_TOKEN=${LOGFIRE_TOKEN:-}" \
-  "LOGFIRE_ENVIRONMENT=$GH_ENV" \
-  --env="$MODAL_ENV" \
-  --force || true
-
 # Sync GCP credentials if provided
 if [ -n "${GCP_CREDENTIALS_JSON:-}" ]; then
   uv run modal secret create gcp-credentials \

.github/workflows/simulation-deploy.reusable.yml

@@ -127,7 +127,6 @@ jobs:
         env:
           MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
           MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
-          LOGFIRE_TOKEN: ${{ secrets.LOGFIRE_TOKEN }}
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
           GCP_CREDENTIALS_JSON: ${{ secrets.GCP_CREDENTIALS_JSON }}
           GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }}
@@ -222,6 +221,10 @@ jobs:
         env:
           IMAGE: ${{ steps.image.outputs.uri }}
           TAG: ${{ steps.metadata.outputs.tag }}
+          OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}
+          OBSERVABILITY_TRACE_PROJECT_ID: ${{ vars.OBSERVABILITY_TRACE_PROJECT_ID }}
+          OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }}
+          POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }}
           DEPLOY_STAGE12_V2: ${{ inputs.deploy_stage12_v2 }}
           APP_ENVIRONMENT: ${{ inputs.deployment_environment }}
           MODAL_ENVIRONMENT: ${{ inputs.modal_environment }}
@@ -292,7 +295,7 @@ jobs:
   deploy_gateway:
     name: Deploy and unit test Modal gateway
     if: ${{ inputs.deploy_existing_stack }}
-    needs: prepare
+    needs: [prepare, deploy_executor]
     runs-on: ubuntu-latest
     environment: ${{ inputs.release_environment }}
     outputs:
@@ -315,6 +318,14 @@ jobs:
         env:
           MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
           MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
+          OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}
+          OBSERVABILITY_TRACE_PROJECT_ID: ${{ vars.OBSERVABILITY_TRACE_PROJECT_ID }}
+          OBSERVABILITY_LOGGING_PROJECT_ID: ${{ vars.OBSERVABILITY_LOGGING_PROJECT_ID }}
+          OBSERVABILITY_LOG_NAME: ${{ vars.OBSERVABILITY_LOG_NAME }}
+          OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
+          OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ vars.OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL }}
+          OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }}
+          POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }}
         run: uv run modal deploy --env="${{ inputs.modal_environment }}" src/policyengine_simulation_gateway/app.py
 
       - name: Run gateway unit tests
@@ -383,6 +394,14 @@ jobs:
           POLICYENGINE_MANIFEST_DIGEST: ${{ steps.precompute.outputs.manifest_digest }}
           POLICYENGINE_ARTIFACT_BUCKET: ${{ vars.POLICYENGINE_ARTIFACT_BUCKET }}
           GCP_CREDENTIALS_JSON: ${{ secrets.GCP_CREDENTIALS_JSON }}
+          OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}
+          OBSERVABILITY_TRACE_PROJECT_ID: ${{ vars.OBSERVABILITY_TRACE_PROJECT_ID }}
+          OBSERVABILITY_LOGGING_PROJECT_ID: ${{ vars.OBSERVABILITY_LOGGING_PROJECT_ID }}
+          OBSERVABILITY_LOG_NAME: ${{ vars.OBSERVABILITY_LOG_NAME }}
+          OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
+          OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ vars.OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL }}
+          OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }}
+          POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }}
         run: uv run modal deploy --env="${{ inputs.modal_environment }}" src/modal/app.py
 
       - name: Run executor unit tests
@@ -543,6 +562,14 @@ jobs:
           STAGE12_EXPECT_US_VERSION: ${{ needs.prepare.outputs.us_version }}
           STAGE12_EXPECT_UK_VERSION: ${{ needs.prepare.outputs.uk_version }}
           STAGE12_ARTIFACT_BUCKET: ${{ vars.STAGE12_ARTIFACT_BUCKET }}
+          OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}
+          OBSERVABILITY_TRACE_PROJECT_ID: ${{ vars.OBSERVABILITY_TRACE_PROJECT_ID }}
+          OBSERVABILITY_LOGGING_PROJECT_ID: ${{ vars.OBSERVABILITY_LOGGING_PROJECT_ID }}
+          OBSERVABILITY_LOG_NAME: ${{ vars.OBSERVABILITY_LOG_NAME }}
+          OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
+          OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ vars.OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL }}
+          OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }}
+          POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }}
         run: uv run modal deploy --env="${{ inputs.modal_environment }}" src/modal/v2_app.py
 
       - name: Validate installed bundles, secrets, datasets, and calculations

libs/policyengine-fastapi/pyproject.toml

@@ -10,14 +10,14 @@ requires-python = ">=3.13"
 dependencies = [
     "fastapi[standard] >=0.115.8,<0.116.0",
     "pyjwt >=2.10.1,<3.0.0",
-    "opentelemetry-sdk >=1.30.0,<2.0.0",
+    "opentelemetry-sdk >=1.44.0,<2.0.0",
     "sqlmodel >=0.0.22,<0.0.23",
     "python-json-logger >=3.2.1,<4.0.0",
-    "opentelemetry-instrumentation-logging >=0.51b0,<0.52",
+    "opentelemetry-instrumentation-logging >=0.65b0,<0.66",
     "opentelemetry-exporter-gcp-trace >=1.9.0,<2.0.0",
     "opentelemetry-exporter-gcp-monitoring >=1.9.0a0,<2.0.0",
-    "opentelemetry-instrumentation-fastapi >=0.51b0,<0.52",
-    "opentelemetry-instrumentation-sqlalchemy>=0.51b0",
+    "opentelemetry-instrumentation-fastapi >=0.65b0,<0.66",
+    "opentelemetry-instrumentation-sqlalchemy>=0.65b0,<0.66",
     "uvicorn>=0.35.0",
 ]
 

libs/policyengine-fastapi/src/policyengine_fastapi/observability/__init__.py

@@ -15,13 +15,12 @@
     SimulationCompositeTraceResponse as SimulationCompositeTraceResponse,
     SimulationLifecycleEvent as SimulationLifecycleEvent,
     SimulationRunSummary as SimulationRunSummary,
-    SimulationTelemetryEnvelope as SimulationTelemetryEnvelope,
     SimulationTimelineEntry as SimulationTimelineEntry,
     TracerArtifactManifest as TracerArtifactManifest,
     VersionStageMetricResponse as VersionStageMetricResponse,
 )
 from .correlation import (
-    generate_run_id as generate_run_id,
+    generate_observability_id as generate_observability_id,
     stable_config_hash as stable_config_hash,
 )
 from .emitters import (

libs/policyengine-fastapi/src/policyengine_fastapi/observability/contracts.py

@@ -13,8 +13,8 @@ class ObservabilityModel(BaseModel):
 
 
 class CorrelatedRunFields(ObservabilityModel):
-    run_id: str
-    process_id: str | None = None
+    observability_id: str
+    submission_claim_id: str | None = None
     job_id: str | None = None
     trace_id: str | None = None
     request_id: str | None = None
@@ -54,19 +54,6 @@ class TracerArtifactManifest(CorrelatedRunFields):
     generated_at: datetime
 
 
-class SimulationTelemetryEnvelope(ObservabilityModel):
-    run_id: str
-    process_id: str | None = None
-    request_id: str | None = None
-    traceparent: str | None = None
-    requested_at: datetime | None = None
-    simulation_kind: str | None = None
-    geography_code: str | None = None
-    geography_type: str | None = None
-    config_hash: str | None = None
-    capture_mode: TracerCaptureMode = TracerCaptureMode.DISABLED
-
-
 class SimulationRunSummary(CorrelatedRunFields):
     status: str
     requested_at: datetime | None = None

libs/policyengine-fastapi/src/policyengine_fastapi/observability/correlation.py

@@ -6,7 +6,7 @@
 from uuid import uuid4
 
 
-def generate_run_id() -> str:
+def generate_observability_id() -> str:
     return str(uuid4())
 
 

libs/policyengine-fastapi/tests/test_observability.py

@@ -7,13 +7,12 @@
     SimulationLifecycleEvent,
     SimulationRunSummary,
     SimulationStage,
-    SimulationTelemetryEnvelope,
     SimulationTimelineEntry,
     TracerArtifactManifest,
     TracerCaptureMode,
     VersionStageMetricResponse,
     build_observability,
-    generate_run_id,
+    generate_observability_id,
     get_observability,
     parse_header_value_pairs,
     stable_config_hash,
@@ -49,11 +48,11 @@ def test_observability_config_disabled__returns_disabled_defaults():
 
 
 def test_correlation_helpers__generate_ids_and_stable_hashes():
-    run_id = generate_run_id()
+    observability_id = generate_observability_id()
     left = stable_config_hash({"b": 2, "a": 1})
     right = stable_config_hash({"a": 1, "b": 2})
 
-    assert len(run_id) == 36
+    assert len(observability_id) == 36
     assert left == right
     assert left.startswith("sha256:")
 
@@ -67,18 +66,18 @@ def test_contract_models__serialize_expected_shapes():
         status="ok",
         timestamp=timestamp,
         service="policyengine-simulation-worker",
-        run_id="run-123",
+        observability_id="run-123",
     )
     manifest = TracerArtifactManifest(
-        run_id="run-123",
+        observability_id="run-123",
         scenario="baseline",
         capture_mode=TracerCaptureMode.THRESHOLD,
         artifact_format="policyengine.flat_trace.v1",
         storage_uri="gs://bucket/run-123/trace.json.gz",
         generated_at=timestamp,
     )
     response = SimulationCompositeTraceResponse(
-        run=SimulationRunSummary(run_id="run-123", status="complete"),
+        run=SimulationRunSummary(observability_id="run-123", status="complete"),
         timeline=[
             SimulationTimelineEntry(
                 stage=SimulationStage.REQUEST_ACCEPTED,
@@ -107,15 +106,6 @@ def test_contract_models__serialize_expected_shapes():
     assert dumped_version_metrics["versions"] == []
 
 
-def test_telemetry_envelope__serializes_expected_defaults():
-    envelope = SimulationTelemetryEnvelope(run_id="run-123")
-
-    dumped = envelope.model_dump(mode="json")
-
-    assert dumped["run_id"] == "run-123"
-    assert dumped["capture_mode"] == "disabled"
-
-
 def test_contract_models__reject_extra_fields():
     try:
         SimulationLifecycleEvent(
@@ -124,7 +114,7 @@ def test_contract_models__reject_extra_fields():
             status="ok",
             timestamp=datetime(2026, 4, 9, 20, 0, tzinfo=UTC),
             service="policyengine-simulation-worker",
-            run_id="run-789",
+            observability_id="run-789",
             unexpected=True,
         )
     except ValidationError as error:
@@ -133,15 +123,6 @@ def test_contract_models__reject_extra_fields():
         raise AssertionError("Expected extra field validation to fail")
 
 
-def test_contract_models__reject_invalid_enum_values():
-    try:
-        SimulationTelemetryEnvelope(run_id="run-123", capture_mode="bad-mode")
-    except ValidationError as error:
-        assert "capture_mode" in str(error)
-    else:
-        raise AssertionError("Expected invalid enum validation to fail")
-
-
 def test_noop_observability__accepts_calls_without_side_effects():
     observability = NoOpObservability()
     event = SimulationLifecycleEvent(
@@ -150,14 +131,14 @@ def test_noop_observability__accepts_calls_without_side_effects():
         status="ok",
         timestamp=datetime(2026, 4, 9, 20, 0, tzinfo=UTC),
         service="policyengine-simulation-worker",
-        run_id="run-456",
+        observability_id="run-456",
     )
 
     observability.emit_lifecycle_event(event)
     observability.emit_counter("policyengine.simulation.run.count")
     observability.emit_histogram("policyengine.simulation.run.duration.seconds", 1.23)
     with observability.span("run_simulation") as span:
-        span.set_attribute("run_id", "run-456")
+        span.set_attribute("observability_id", "run-456")
         span.add_event("simulation.completed")
     observability.flush()