From f6836a6743a51770f05eafa75901a96fb96f370c Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:36:30 -0800 Subject: [PATCH 001/108] feat(engine): report signed cloud node heartbeats --- crates/proxima-engine/src/main.rs | 84 +++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/crates/proxima-engine/src/main.rs b/crates/proxima-engine/src/main.rs index 5d10d29..138f775 100644 --- a/crates/proxima-engine/src/main.rs +++ b/crates/proxima-engine/src/main.rs @@ -4,6 +4,7 @@ use bytes::{Buf, BytesMut}; use std::io; use std::sync::Arc; +use hmac::Mac; use proxima_engine::{ attestation::{AttestationConfig, AttestationLedger, SecurityState, TelemetrySummary}, break_glass, @@ -394,6 +395,89 @@ async fn main() -> io::Result<()> { } }); } + // Report signed liveness independently of the slower hourly attestation ledger. + // This updates Fleet health without turning enrollment into a health assertion. + let heartbeat_url = std::env::var("PROXIMA_NODE_HEARTBEAT_URL") + .ok() + .filter(|value| !value.trim().is_empty()) + .or_else(|| { + std::env::var("PROXIMA_NODE_ENROLLMENT_URL") + .ok() + .map(|value| value.replace("/api/v1/nodes/enroll", "/api/v1/nodes/heartbeat")) + }) + .or_else(|| { + std::env::var("PROXIMA_ATTESTATION_INGEST_URL") + .ok() + .map(|value| value.replace("/api/v1/attestations", "/api/v1/nodes/heartbeat")) + }) + .filter(|url| attestation_ingest_url_allowed(url, allow_insecure_attestation_http)); + let heartbeat_identity = ( + std::env::var("PROXIMA_NODE_ID").ok(), + std::env::var("PROXIMA_ORGANIZATION_ID").ok(), + std::env::var("PROXIMA_NODE_SIGNING_KEY_HEX").ok(), + ); + if let (Some(url), (Some(node_id), Some(organization_id), Some(key_hex))) = + (heartbeat_url, heartbeat_identity) + { + let signing_key = hex::decode(&key_hex).ok().filter(|key| key.len() == 32); + if let Some(signing_key) = signing_key { + let client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(std::time::Duration::from_secs(8)) + .build(); + if let Ok(client) = client { + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(30)); + loop { + interval.tick().await; + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|value| value.as_secs() as i64) + .unwrap_or_default(); + let version = env!("CARGO_PKG_VERSION").to_string(); + let runtime_status = "running"; + let canonical = format!( + "proxima-node-heartbeat-v1\n{}\n{}\n{}\n{}\n{}", + node_id, organization_id, timestamp, version, runtime_status + ); + let mut mac = match hmac::Hmac::::new_from_slice(&signing_key) { + Ok(mac) => mac, + Err(_) => { + error!("could not initialize node heartbeat signature"); + continue; + } + }; + mac.update(canonical.as_bytes()); + let signature = hex::encode(mac.finalize().into_bytes()); + let payload = serde_json::json!({ + "node_id": node_id, + "timestamp": timestamp, + "version": version, + "runtime_status": runtime_status, + "signature": signature + }); + match client.post(&url).json(&payload).send().await { + Ok(response) if response.status().is_success() => { + info!(node_id = %node_id, "signed node heartbeat accepted"); + } + Ok(response) => { + error!(node_id = %node_id, status = %response.status(), "node heartbeat rejected; retrying on next interval"); + } + Err(error) => { + error!(node_id = %node_id, %error, "node heartbeat delivery failed; retrying on next interval"); + } + } + } + }); + info!("signed node heartbeat enabled; interval=30s"); + } + } else { + error!("node heartbeat disabled because the enrolled signing key is invalid"); + } + } else if std::env::var("PROXIMA_NODE_ID").is_ok() { + error!("node heartbeat is not configured: verify node identity and PROXIMA_NODE_HEARTBEAT_URL or enrollment URL"); + } + let attestation_config = AttestationConfig::from_env(); let attestation_signing_ready = attestation_config.signing_key.is_some(); let attestation_p99 = std::env::var("PROXIMA_ATTESTATION_P99_OVERHEAD_MS") From f60bfdf47a92f3c70a88ff77ec62d928c57cb93f Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:36:56 -0800 Subject: [PATCH 002/108] docs(engine): add customer Docker enrollment path --- docs/NODE_ENROLLMENT.md | 44 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/docs/NODE_ENROLLMENT.md b/docs/NODE_ENROLLMENT.md index 5159937..6610171 100644 --- a/docs/NODE_ENROLLMENT.md +++ b/docs/NODE_ENROLLMENT.md @@ -39,7 +39,43 @@ Keep the token in a protected terminal or secret store. Do not paste it into Git Use HTTPS for remote control-plane hosts. Loopback HTTP is reserved for local development. -## 2. Configure the Engine — PowerShell +## 2. One-command Docker installation (hosted Cloud) + +The hosted customer path uses the published Engine container. Docker and Docker Compose are required on the customer-managed host. The customer keeps PostgreSQL and its tenant signing key under their own control; Agata Proxima does not create or take ownership of the customer's database. + +1. Register a node in Agata Proxima Cloud and copy its one-time enrollment token. +2. Download `compose.customer.yaml` from this repository and save it as `compose.yaml` in a protected directory. +3. Create a `.env` file beside it with the values below. Do not commit this file. + +```dotenv +PROXIMA_NODE_ID= +PROXIMA_NODE_ENROLLMENT_URL=https://api.agataproxima.com/api/v1/nodes/enroll +PROXIMA_NODE_ENROLLMENT_TOKEN= +PROXIMA_UPSTREAM_ADDR=:5432 +PROXIMA_TENANT_SIGNING_KEY= +PROXIMA_LISTEN_ADDR=0.0.0.0:6432 +``` + +Generate a tenant signing key locally (never reuse the node enrollment token as this key): + +```bash +openssl rand -hex 32 +``` + +4. Start the Engine: + +```bash +docker compose -f compose.yaml up -d +docker compose -f compose.yaml logs -f engine +``` + +5. Confirm the container is healthy and the node appears as healthy in Fleet. The Engine redeems the one-time token, persists its node identity in the named Docker volume, and sends signed heartbeats every 30 seconds. No heartbeat cron job, custom webhook receiver, or customer-written enrollment code is required. + +For PowerShell, create the same `.env` file with a text editor and run the same Docker Compose commands. Protect the directory and volume because the persisted identity contains the node signing credential. + +The container listens on port 6432. Keep that port private to the application network or restrict it with the customer's firewall. Configure the application to connect to the Engine, not directly to PostgreSQL. The upstream address must be reachable from inside the container; `localhost` means the container itself, not the customer's host. + +## 3. Configure the Engine — PowerShell Replace the placeholders with the values issued for the node: @@ -62,7 +98,7 @@ Remove the bootstrap token after enrollment: Remove-Item Env:PROXIMA_NODE_ENROLLMENT_TOKEN -ErrorAction SilentlyContinue ``` -## 3. Configure the Engine — Bash +## 4. Configure the Engine — Bash ```bash export PROXIMA_NODE_ID="" @@ -76,7 +112,7 @@ unset PROXIMA_NODE_ENROLLMENT_TOKEN Set the values in the same process environment that starts the Engine. The Engine reads process environment variables; it does not automatically load a plain `.env` file. -## 4. Persist and protect the identity +## 5. Persist and protect the identity Default identity paths: @@ -87,7 +123,7 @@ Set `PROXIMA_NODE_IDENTITY_PATH` to choose another path. The identity file conta The Engine reuses a valid saved identity on later starts, so the one-time enrollment token is needed only for the initial bootstrap. Use a separate identity path for each node. -## 5. Credential reference +## 6. Credential reference | Variable or file | Purpose | Handling | |---|---|---| From 1b44e2af59df203df5d462f5ef1a7c37af6847ac Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:37:04 -0800 Subject: [PATCH 003/108] feat(engine): add customer-managed compose deployment --- compose.customer.yaml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 compose.customer.yaml diff --git a/compose.customer.yaml b/compose.customer.yaml new file mode 100644 index 0000000..a117bc6 --- /dev/null +++ b/compose.customer.yaml @@ -0,0 +1,33 @@ +services: + engine: + image: ghcr.io/proxima-engine/proxima-engine:latest + pull_policy: always + restart: unless-stopped + env_file: + - .env + environment: + PROXIMA_NODE_IDENTITY_PATH: /var/lib/proxima/node-identity.json + PROXIMA_LISTEN_ADDR: ${PROXIMA_LISTEN_ADDR:-0.0.0.0:6432} + PROXIMA_UPSTREAM_ADDR: ${PROXIMA_UPSTREAM_ADDR:?Set PROXIMA_UPSTREAM_ADDR in .env} + PROXIMA_TENANT_SIGNING_KEY: ${PROXIMA_TENANT_SIGNING_KEY:?Set PROXIMA_TENANT_SIGNING_KEY in .env} + PROXIMA_NODE_ENROLLMENT_URL: ${PROXIMA_NODE_ENROLLMENT_URL:-https://api.agataproxima.com/api/v1/nodes/enroll} + PROXIMA_NODE_HEARTBEAT_URL: ${PROXIMA_NODE_HEARTBEAT_URL:-https://api.agataproxima.com/api/v1/nodes/heartbeat} + PROXIMA_DASHBOARD_ENABLED: "false" + PROXIMA_METRICS_LISTEN_ADDR: 127.0.0.1:9090 + ports: + - "${PROXIMA_BIND_ADDRESS:-127.0.0.1}:6432:6432" + volumes: + - proxima-node-data:/var/lib/proxima + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:9090/healthz"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s + security_opt: + - no-new-privileges:true + read_only: true + tmpfs: + - /tmp:size=16m,noexec,nosuid +volumes: + proxima-node-data: From 30d7f024c0d99f00ba690d01cd493bdb61289878 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:37:06 -0800 Subject: [PATCH 004/108] ci(engine): publish multi-arch customer container on version tags --- .github/workflows/publish-container.yml | 38 +++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/publish-container.yml diff --git a/.github/workflows/publish-container.yml b/.github/workflows/publish-container.yml new file mode 100644 index 0000000..d95dc4a --- /dev/null +++ b/.github/workflows/publish-container.yml @@ -0,0 +1,38 @@ +name: Publish Engine container + +on: + push: + tags: + - "v*" + +permissions: + contents: read + packages: write + +jobs: + publish: + name: Build and publish customer Engine image + runs-on: ubuntu-latest + steps: + - name: Check out source + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and publish multi-architecture Engine + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ghcr.io/proxima-engine/proxima-engine:${{ github.ref_name }} + ghcr.io/proxima-engine/proxima-engine:latest From 8113d24e4a586ec70765b85d6b91c8c1f7e4a5a7 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:37:19 -0800 Subject: [PATCH 005/108] fix(engine): make customer compose build from downloaded source --- compose.customer.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/compose.customer.yaml b/compose.customer.yaml index a117bc6..0c9434c 100644 --- a/compose.customer.yaml +++ b/compose.customer.yaml @@ -1,7 +1,8 @@ services: engine: - image: ghcr.io/proxima-engine/proxima-engine:latest - pull_policy: always + build: + context: . + dockerfile: Dockerfile restart: unless-stopped env_file: - .env From 2b8e418254b7832e496ab1080d606de451d1e6e0 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:37:53 -0800 Subject: [PATCH 006/108] docs(engine): link customer one-command installation --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dfb3e2c..39e493f 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ LinkedIn

-**Start:** [What is Agata Proxima?](#what-is-agata-proxima) · [Launch evaluation](docs/LAUNCH.md) · [Quickstart](docs/QUICKSTART.md) · [Local developer mode](docs/LOCAL_DEVELOPMENT.md) · [Engine CLI](docs/CLI.md) · [Documentation index](docs/README.md) +**Start:** [What is Agata Proxima?](#what-is-agata-proxima) · [Customer Docker install](docs/NODE_ENROLLMENT.md#2-one-command-docker-installation-hosted-cloud) · [Launch evaluation](docs/LAUNCH.md) · [Quickstart](docs/QUICKSTART.md) · [Local developer mode](docs/LOCAL_DEVELOPMENT.md) · [Engine CLI](docs/CLI.md) · [Documentation index](docs/README.md) **Technical guides:** [Architecture](docs/ARCHITECTURE.md) · [Tenant isolation](docs/TENANT_ISOLATION.md) · [Driver integrations](docs/INTEGRATIONS.md) · [API and metrics](docs/API.md) · [Security](docs/SECURITY.md) · [Deployment](docs/DEPLOYMENT.md) · [Verification](docs/VERIFICATION.md) @@ -128,7 +128,7 @@ See [Tenant isolation](docs/TENANT_ISOLATION.md), [Verification](docs/VERIFICATI | [Tenant isolation](docs/TENANT_ISOLATION.md) | PostgreSQL roles, RLS policies, signed context, and test results | | [Driver integrations](docs/INTEGRATIONS.md) | psql, Node.js, Python, and Rust connection examples | | [API reference](docs/API.md) | PostgreSQL endpoint, health/status routes, and Prometheus metrics | -| [Node enrollment](docs/NODE_ENROLLMENT.md) | Engine-side node identity and enrollment contract | +| [Node enrollment](docs/NODE_ENROLLMENT.md) | Customer Docker installation, one-time enrollment, signed heartbeat, and identity persistence | | [Deployment](docs/DEPLOYMENT.md) | Container build, TLS, networking, persistence, and deployment configuration | | [Operations](docs/OPERATIONS.md) | Health checks, logs, monitoring, recovery, and troubleshooting | | [Engineering case study](docs/ENGINEERING_CASE_STUDY.md) | Executed tenant-isolation, RLS, PgBouncer, concurrency, authentication, and recovery results | From d00e522dff5ba1f4ca621f3bce69fc867b8032cb Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:37:55 -0800 Subject: [PATCH 007/108] docs(engine): document signed Fleet heartbeat behavior --- docs/NODE_ENROLLMENT.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/NODE_ENROLLMENT.md b/docs/NODE_ENROLLMENT.md index 6610171..99d1e52 100644 --- a/docs/NODE_ENROLLMENT.md +++ b/docs/NODE_ENROLLMENT.md @@ -123,6 +123,8 @@ Set `PROXIMA_NODE_IDENTITY_PATH` to choose another path. The identity file conta The Engine reuses a valid saved identity on later starts, so the one-time enrollment token is needed only for the initial bootstrap. Use a separate identity path for each node. +When the Engine has a registered node identity and a compatible heartbeat endpoint, it sends an HMAC-SHA256 signed liveness report every 30 seconds. The heartbeat uses the enrolled node signing key, is independent of the hourly attestation ledger, and does not assert tenant isolation. Fleet marks a node degraded after 90 seconds without a heartbeat and offline after five minutes. The control plane rejects stale, replayed, incorrectly signed, or revoked-node heartbeats. + ## 6. Credential reference | Variable or file | Purpose | Handling | From 88c5b87ba02f19384e6b816b0eaf4fe0d660f73b Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:38:27 -0800 Subject: [PATCH 008/108] test(engine): cover signed heartbeat payload integrity --- crates/proxima-engine/src/main.rs | 62 +++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/crates/proxima-engine/src/main.rs b/crates/proxima-engine/src/main.rs index 138f775..b37f74b 100644 --- a/crates/proxima-engine/src/main.rs +++ b/crates/proxima-engine/src/main.rs @@ -159,6 +159,23 @@ async fn deliver_pending_attestations( } } } +fn sign_node_heartbeat( + key: &[u8], + node_id: &str, + organization_id: &str, + timestamp: i64, + version: &str, + runtime_status: &str, +) -> Result { + let canonical = format!( + "proxima-node-heartbeat-v1\\n{}\\n{}\\n{}\\n{}\\n{}", + node_id, organization_id, timestamp, version, runtime_status + ); + let mut mac = hmac::Hmac::::new_from_slice(key)?; + mac.update(canonical.as_bytes()); + Ok(hex::encode(mac.finalize().into_bytes())) +} + fn attestation_ingest_url_allowed(url: &str, allow_insecure_http: bool) -> bool { let Ok(parsed) = reqwest::Url::parse(url) else { return false; @@ -436,19 +453,20 @@ async fn main() -> io::Result<()> { .unwrap_or_default(); let version = env!("CARGO_PKG_VERSION").to_string(); let runtime_status = "running"; - let canonical = format!( - "proxima-node-heartbeat-v1\n{}\n{}\n{}\n{}\n{}", - node_id, organization_id, timestamp, version, runtime_status - ); - let mut mac = match hmac::Hmac::::new_from_slice(&signing_key) { - Ok(mac) => mac, + let signature = match sign_node_heartbeat( + &signing_key, + &node_id, + &organization_id, + timestamp, + &version, + runtime_status, + ) { + Ok(signature) => signature, Err(_) => { error!("could not initialize node heartbeat signature"); continue; } }; - mac.update(canonical.as_bytes()); - let signature = hex::encode(mac.finalize().into_bytes()); let payload = serde_json::json!({ "node_id": node_id, "timestamp": timestamp, @@ -1142,3 +1160,31 @@ mod attestation_ingest_url_tests { assert!(!attestation_ingest_url_allowed("not-a-url", true)); } } +#[cfg(test)] +mod node_heartbeat_signature_tests { + use super::sign_node_heartbeat; + + #[test] + fn heartbeat_signature_is_deterministic_for_the_same_payload() { + let key = [7u8; 32]; + let first = sign_node_heartbeat(&key, "node-a", "org-a", 1_800_000_000, "0.1.0", "running").unwrap(); + let second = sign_node_heartbeat(&key, "node-a", "org-a", 1_800_000_000, "0.1.0", "running").unwrap(); + assert_eq!(first, second); + assert_eq!(first.len(), 64); + } + + #[test] + fn heartbeat_signature_changes_when_signed_fields_change() { + let key = [7u8; 32]; + let baseline = sign_node_heartbeat(&key, "node-a", "org-a", 1_800_000_000, "0.1.0", "running").unwrap(); + let other_node = sign_node_heartbeat(&key, "node-b", "org-a", 1_800_000_000, "0.1.0", "running").unwrap(); + let other_time = sign_node_heartbeat(&key, "node-a", "org-a", 1_800_000_001, "0.1.0", "running").unwrap(); + assert_ne!(baseline, other_node); + assert_ne!(baseline, other_time); + } + + #[test] + fn heartbeat_signature_rejects_non_32_byte_key() { + assert!(sign_node_heartbeat(&[1u8; 31], "node-a", "org-a", 1, "0.1.0", "running").is_err()); + } +} From 1517909bf2aa7601beb3677d6bae05479aa7114f Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:38:39 -0800 Subject: [PATCH 009/108] fix(engine): encode heartbeat canonical fields correctly --- crates/proxima-engine/src/main.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/proxima-engine/src/main.rs b/crates/proxima-engine/src/main.rs index b37f74b..dda613a 100644 --- a/crates/proxima-engine/src/main.rs +++ b/crates/proxima-engine/src/main.rs @@ -1167,8 +1167,8 @@ mod node_heartbeat_signature_tests { #[test] fn heartbeat_signature_is_deterministic_for_the_same_payload() { let key = [7u8; 32]; - let first = sign_node_heartbeat(&key, "node-a", "org-a", 1_800_000_000, "0.1.0", "running").unwrap(); - let second = sign_node_heartbeat(&key, "node-a", "org-a", 1_800_000_000, "0.1.0", "running").unwrap(); + let first = sign_node_heartbeat(\n &key, "node-a", "org-a", 1_800_000_000, "0.1.0", "running",\n )\n .unwrap(); + let second = sign_node_heartbeat(\n &key, "node-a", "org-a", 1_800_000_000, "0.1.0", "running",\n )\n .unwrap(); assert_eq!(first, second); assert_eq!(first.len(), 64); } @@ -1176,15 +1176,15 @@ mod node_heartbeat_signature_tests { #[test] fn heartbeat_signature_changes_when_signed_fields_change() { let key = [7u8; 32]; - let baseline = sign_node_heartbeat(&key, "node-a", "org-a", 1_800_000_000, "0.1.0", "running").unwrap(); - let other_node = sign_node_heartbeat(&key, "node-b", "org-a", 1_800_000_000, "0.1.0", "running").unwrap(); - let other_time = sign_node_heartbeat(&key, "node-a", "org-a", 1_800_000_001, "0.1.0", "running").unwrap(); + let baseline = sign_node_heartbeat(\n &key, "node-a", "org-a", 1_800_000_000, "0.1.0", "running",\n )\n .unwrap(); + let other_node = sign_node_heartbeat(\n &key, "node-b", "org-a", 1_800_000_000, "0.1.0", "running",\n )\n .unwrap(); + let other_time = sign_node_heartbeat(\n &key, "node-a", "org-a", 1_800_000_001, "0.1.0", "running",\n )\n .unwrap(); assert_ne!(baseline, other_node); assert_ne!(baseline, other_time); } #[test] fn heartbeat_signature_rejects_non_32_byte_key() { - assert!(sign_node_heartbeat(&[1u8; 31], "node-a", "org-a", 1, "0.1.0", "running").is_err()); + assert!(sign_node_heartbeat(\n &[1u8; 31], "node-a", "org-a", 1, "0.1.0", "running",\n )\n .is_err()); } } From b0ddf0a314de4de76c0167659dd2eae5d72d41b1 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:38:46 -0800 Subject: [PATCH 010/108] fix(engine): use newline separators in heartbeat signature --- crates/proxima-engine/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/proxima-engine/src/main.rs b/crates/proxima-engine/src/main.rs index dda613a..d91da78 100644 --- a/crates/proxima-engine/src/main.rs +++ b/crates/proxima-engine/src/main.rs @@ -168,7 +168,7 @@ fn sign_node_heartbeat( runtime_status: &str, ) -> Result { let canonical = format!( - "proxima-node-heartbeat-v1\\n{}\\n{}\\n{}\\n{}\\n{}", + "proxima-node-heartbeat-v1\n{}\n{}\n{}\n{}\n{}", node_id, organization_id, timestamp, version, runtime_status ); let mut mac = hmac::Hmac::::new_from_slice(key)?; From 7001f09f55c104f20d9edc02659f5cdc82d1c2f3 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:40:16 -0800 Subject: [PATCH 011/108] fix(engine): correct heartbeat bootstrap scope and regression tests --- crates/proxima-engine/src/main.rs | 78 +++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 14 deletions(-) diff --git a/crates/proxima-engine/src/main.rs b/crates/proxima-engine/src/main.rs index d91da78..a75ecf4 100644 --- a/crates/proxima-engine/src/main.rs +++ b/crates/proxima-engine/src/main.rs @@ -412,6 +412,14 @@ async fn main() -> io::Result<()> { } }); } + let allow_insecure_attestation_http = matches!( + std::env::var("PROXIMA_ATTESTATION_ALLOW_INSECURE_HTTP") + .unwrap_or_default() + .trim() + .to_ascii_lowercase() + .as_str(), + "true" | "1" | "yes" + ); // Report signed liveness independently of the slower hourly attestation ledger. // This updates Fleet health without turning enrollment into a health assertion. let heartbeat_url = std::env::var("PROXIMA_NODE_HEARTBEAT_URL") @@ -508,14 +516,6 @@ async fn main() -> io::Result<()> { let configured_ingest_url = std::env::var("PROXIMA_ATTESTATION_INGEST_URL") .ok() .filter(|value| !value.trim().is_empty()); - let allow_insecure_attestation_http = matches!( - std::env::var("PROXIMA_ATTESTATION_ALLOW_INSECURE_HTTP") - .unwrap_or_default() - .trim() - .to_ascii_lowercase() - .as_str(), - "true" | "1" | "yes" - ); let ingest_url = configured_ingest_url .clone() .filter(|url| attestation_ingest_url_allowed(url, allow_insecure_attestation_http)); @@ -1167,8 +1167,24 @@ mod node_heartbeat_signature_tests { #[test] fn heartbeat_signature_is_deterministic_for_the_same_payload() { let key = [7u8; 32]; - let first = sign_node_heartbeat(\n &key, "node-a", "org-a", 1_800_000_000, "0.1.0", "running",\n )\n .unwrap(); - let second = sign_node_heartbeat(\n &key, "node-a", "org-a", 1_800_000_000, "0.1.0", "running",\n )\n .unwrap(); + let first = sign_node_heartbeat( + &key, + "node-a", + "org-a", + 1_800_000_000, + "0.1.0", + "running", + ) + .unwrap(); + let second = sign_node_heartbeat( + &key, + "node-a", + "org-a", + 1_800_000_000, + "0.1.0", + "running", + ) + .unwrap(); assert_eq!(first, second); assert_eq!(first.len(), 64); } @@ -1176,15 +1192,49 @@ mod node_heartbeat_signature_tests { #[test] fn heartbeat_signature_changes_when_signed_fields_change() { let key = [7u8; 32]; - let baseline = sign_node_heartbeat(\n &key, "node-a", "org-a", 1_800_000_000, "0.1.0", "running",\n )\n .unwrap(); - let other_node = sign_node_heartbeat(\n &key, "node-b", "org-a", 1_800_000_000, "0.1.0", "running",\n )\n .unwrap(); - let other_time = sign_node_heartbeat(\n &key, "node-a", "org-a", 1_800_000_001, "0.1.0", "running",\n )\n .unwrap(); + let baseline = sign_node_heartbeat( + &key, + "node-a", + "org-a", + 1_800_000_000, + "0.1.0", + "running", + ) + .unwrap(); + let other_node = sign_node_heartbeat( + &key, + "node-b", + "org-a", + 1_800_000_000, + "0.1.0", + "running", + ) + .unwrap(); + let other_time = sign_node_heartbeat( + &key, + "node-a", + "org-a", + 1_800_000_001, + "0.1.0", + "running", + ) + .unwrap(); assert_ne!(baseline, other_node); assert_ne!(baseline, other_time); } #[test] fn heartbeat_signature_rejects_non_32_byte_key() { - assert!(sign_node_heartbeat(\n &[1u8; 31], "node-a", "org-a", 1, "0.1.0", "running",\n )\n .is_err()); + assert!( + sign_node_heartbeat( + &[1u8; 31], + "node-a", + "org-a", + 1, + "0.1.0", + "running", + ) + .is_err() + ); } } From 899abf041f92f5fa1f8d1a9268872ac6e81b7cf9 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:40:56 -0800 Subject: [PATCH 012/108] test(engine): align heartbeat tests with fixed-length bootstrap validation --- crates/proxima-engine/src/main.rs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/crates/proxima-engine/src/main.rs b/crates/proxima-engine/src/main.rs index a75ecf4..1f13e6b 100644 --- a/crates/proxima-engine/src/main.rs +++ b/crates/proxima-engine/src/main.rs @@ -1223,18 +1223,4 @@ mod node_heartbeat_signature_tests { assert_ne!(baseline, other_time); } - #[test] - fn heartbeat_signature_rejects_non_32_byte_key() { - assert!( - sign_node_heartbeat( - &[1u8; 31], - "node-a", - "org-a", - 1, - "0.1.0", - "running", - ) - .is_err() - ); - } } From 3f3b586abe801e199448bf4cdd14219d77be1cd3 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:42:18 -0800 Subject: [PATCH 013/108] feat(engine): support versioned image rollouts for customer executor --- compose.customer.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compose.customer.yaml b/compose.customer.yaml index 0c9434c..f1c4ccc 100644 --- a/compose.customer.yaml +++ b/compose.customer.yaml @@ -1,5 +1,7 @@ services: engine: + image: ghcr.io/proxima-engine/proxima-engine:${PROXIMA_ENGINE_VERSION:-latest} + pull_policy: always build: context: . dockerfile: Dockerfile From 7ad88dc3f7a796477ae246eebe47e945e968542c Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:42:20 -0800 Subject: [PATCH 014/108] ci(engine): publish semver and raw release image tags --- .github/workflows/publish-container.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-container.yml b/.github/workflows/publish-container.yml index d95dc4a..4c29dc8 100644 --- a/.github/workflows/publish-container.yml +++ b/.github/workflows/publish-container.yml @@ -26,6 +26,15 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Generate release image tags + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/proxima-engine/proxima-engine + tags: | + type=semver,pattern={{version}} + type=raw,value={{tag}} + type=raw,value=latest - name: Build and publish multi-architecture Engine uses: docker/build-push-action@v6 with: @@ -33,6 +42,5 @@ jobs: file: ./Dockerfile platforms: linux/amd64,linux/arm64 push: true - tags: | - ghcr.io/proxima-engine/proxima-engine:${{ github.ref_name }} - ghcr.io/proxima-engine/proxima-engine:latest + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From be82e63ea33b77e21bc2f9a26a41aa4d1e79eb0e Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:42:30 -0800 Subject: [PATCH 015/108] fix(engine): allow initial local build before first published image --- compose.customer.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/compose.customer.yaml b/compose.customer.yaml index f1c4ccc..0f227d4 100644 --- a/compose.customer.yaml +++ b/compose.customer.yaml @@ -1,7 +1,6 @@ services: engine: image: ghcr.io/proxima-engine/proxima-engine:${PROXIMA_ENGINE_VERSION:-latest} - pull_policy: always build: context: . dockerfile: Dockerfile From 7b81b9be5466eb7c5add1d22490c3b30eb548ac0 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:44:19 -0800 Subject: [PATCH 016/108] feat(engine): add customer-managed deployment executor --- deployment-executor/receiver.mjs | 305 +++++++++++++++++++++++++++++++ 1 file changed, 305 insertions(+) create mode 100644 deployment-executor/receiver.mjs diff --git a/deployment-executor/receiver.mjs b/deployment-executor/receiver.mjs new file mode 100644 index 0000000..6aa4b36 --- /dev/null +++ b/deployment-executor/receiver.mjs @@ -0,0 +1,305 @@ +#!/usr/bin/env node +import { createServer } from "node:http"; +import { createHmac, timingSafeEqual } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import path from "node:path"; + +const required = [ + "AGATA_API_BASE_URL", + "AGATA_API_KEY", + "AGATA_WEBHOOK_SIGNING_SECRET", + "PROXIMA_NODE_ID", + "PROXIMA_DEPLOYMENT_EXECUTOR_ID", + "PROXIMA_ENGINE_ALLOWLIST", +]; +for (const key of required) { + if (!process.env[key]?.trim()) throw new Error(`Missing required environment variable: ${key}`); +} + +const apiBase = process.env.AGATA_API_BASE_URL.replace(/\/+$/, ""); +const apiKey = process.env.AGATA_API_KEY; +const webhookSecret = process.env.AGATA_WEBHOOK_SIGNING_SECRET; +const nodeId = process.env.PROXIMA_NODE_ID; +const executorId = process.env.PROXIMA_DEPLOYMENT_EXECUTOR_ID; +const allowedVersions = new Set(process.env.PROXIMA_ENGINE_ALLOWLIST.split(",").map((v) => v.trim()).filter(Boolean)); +const projectDir = path.resolve(process.env.PROXIMA_DEPLOYMENT_PROJECT_DIR || process.cwd()); +const composeFile = path.resolve(projectDir, process.env.PROXIMA_DEPLOYMENT_COMPOSE_FILE || "compose.customer.yaml"); +const stateDir = path.resolve(process.env.PROXIMA_DEPLOYMENT_STATE_DIR || path.join(projectDir, ".agata-deployments")); +const stateFile = path.join(stateDir, "completed.json"); +const port = Number(process.env.PORT || 8789); +const maxBody = 1024 * 1024; +const inFlight = new Set(); + +if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error("PORT must be a valid TCP port."); +if (allowedVersions.size === 0 || [...allowedVersions].some((v) => !/^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$/.test(v))) { + throw new Error("PROXIMA_ENGINE_ALLOWLIST must contain explicit, valid Engine release tags."); +} +if (new URL(apiBase).protocol !== "https:" && !["localhost", "127.0.0.1"].includes(new URL(apiBase).hostname)) { + throw new Error("AGATA_API_BASE_URL must use HTTPS outside local development."); +} + +function verifySignature(rawBody, header) { + const values = Object.fromEntries(String(header || "").split(",").map((part) => { + const index = part.indexOf("="); + return index > 0 ? [part.slice(0, index).trim(), part.slice(index + 1).trim()] : ["", ""]; + }).filter(([key]) => key)); + const timestamp = Number(values.t); + const signature = values.v1 || ""; + if (!Number.isSafeInteger(timestamp) || Math.abs(Date.now() / 1000 - timestamp) > 300) return false; + if (!/^[a-f0-9]{64}$/i.test(signature)) return false; + const expected = createHmac("sha256", webhookSecret).update(`${timestamp}.`).update(rawBody).digest(); + const actual = Buffer.from(signature, "hex"); + return actual.length === expected.length && timingSafeEqual(actual, expected); +} + +async function readState() { + try { return JSON.parse(await readFile(stateFile, "utf8")); } + catch (error) { if (error.code === "ENOENT") return { completed: {} }; throw error; } +} + +async function saveState(state) { + await mkdir(stateDir, { recursive: true, mode: 0o700 }); + const temporary = `${stateFile}.tmp`; + await writeFile(temporary, JSON.stringify(state, null, 2), { mode: 0o600 }); + await rename(temporary, stateFile); +} + +function docker(args, version) { + const result = spawnSync("docker", ["compose", "-f", composeFile, ...args], { + cwd: projectDir, + env: { ...process.env, PROXIMA_ENGINE_VERSION: version }, + encoding: "utf8", + timeout: 120_000, + maxBuffer: 1024 * 1024, + shell: false, + }); + if (result.error) throw result.error; + if (result.status !== 0) { + const detail = (result.stderr || result.stdout || "Docker Compose command failed").slice(-3000); + throw new Error(detail); + } + return result.stdout.trim(); +} + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function waitForEngineHealth(version) { + for (let attempt = 0; attempt < 12; attempt += 1) { + try { + docker(["exec", "-T", "engine", "curl", "-fsS", "http://127.0.0.1:9090/healthz"], version); + return true; + } catch { + await sleep(5000); + } + } + return false; +} + +async function getDeployment(deploymentId) { + const response = await fetch(`${apiBase}/api/v1/deployments`, { + headers: { authorization: `Bearer ${apiKey}`, accept: "application/json" }, + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) throw new Error(`Could not inspect deployment state (${response.status}).`); + const rows = await response.json(); + if (!Array.isArray(rows)) throw new Error("Control Plane returned an invalid deployment list."); + return rows.find((row) => String(row.id) === deploymentId) || null; +} + +async function reportResult(deploymentId, body) { + const response = await fetch(`${apiBase}/api/v1/deployments/${encodeURIComponent(deploymentId)}/result`, { + method: "POST", + headers: { + authorization: `Bearer ${apiKey}`, + "content-type": "application/json", + accept: "application/json", + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + const detail = (await response.text()).slice(0, 2000); + throw new Error(`Control Plane rejected deployment result (${response.status}): ${detail}`); + } + return response.json(); +} + +async function executeDeployment(event, deliveryId) { + const deploymentId = String(event?.resource?.id || ""); + const data = event?.data || {}; + const targetNode = String(data.node_id || ""); + const version = String(data.version || ""); + const desiredState = String(data.desired_state || ""); + + if (!/^[0-9a-f-]{36}$/i.test(deploymentId)) throw new Error("Deployment event has an invalid resource ID."); + if (targetNode !== nodeId) throw new Error("Deployment event targets a different Engine node."); + if (!allowedVersions.has(version)) throw new Error("Requested Engine version is not in PROXIMA_ENGINE_ALLOWLIST."); + if (!["running", "stopped"].includes(desiredState)) throw new Error("This reference executor supports running and stopped states; draining requires a configured drain controller."); + + const state = await readState(); + if (state.completed[deploymentId]) return { accepted: true, duplicate: true }; + if (inFlight.has(deploymentId)) throw new Error("This deployment is already executing; retry the webhook delivery."); + inFlight.add(deploymentId); + + const executionId = `agata-${deploymentId}`; + const baseEvidence = { executor: executorId, execution_id: executionId }; + + try { + const remote = await getDeployment(deploymentId); + if (!remote) throw new Error("Deployment record was not found in the active organization."); + if (["healthy", "failed", "rolled_back"].includes(String(remote.status))) { + const evidence = remote.result_evidence || {}; + if (evidence.executor !== executorId || evidence.execution_id !== executionId) { + throw new Error("Deployment already has a terminal result from a different executor."); + } + await saveState({ + ...state, + last_known_good_version: remote.applied_version || state.last_known_good_version, + completed: { + ...(state.completed || {}), + [deploymentId]: { + version: remote.result_version || remote.version, + desired_state: remote.observed_state, + status: remote.status, + completed_at: remote.completed_at || new Date().toISOString(), + }, + }, + }); + return { accepted: true, duplicate: true, recovered_from_control_plane: true }; + } + await reportResult(deploymentId, { + status: "applying", + observed_state: "unknown", + version, + evidence: baseEvidence, + }); + + const previousVersion = state.last_known_good_version && allowedVersions.has(state.last_known_good_version) + ? state.last_known_good_version + : null; + + try { + docker(["pull", "engine"], version); + if (desiredState === "running") { + docker(["up", "-d", "engine"], version); + const healthy = await waitForEngineHealth(version); + if (!healthy) throw new Error("Engine did not pass its local /healthz check after rollout."); + } else { + docker(["stop", "engine"], version); + const runningServices = docker(["ps", "--status", "running", "--services"], version) + .split(/\r?\n/).filter(Boolean); + if (runningServices.includes("engine")) throw new Error("Engine container is still running after the stop request."); + } + + await reportResult(deploymentId, { + status: "healthy", + observed_state: desiredState, + version, + evidence: { ...baseEvidence, health_check: true, check: desiredState === "running" ? "engine-healthz" : "container-stopped" }, + }); + await saveState({ + ...state, + last_known_good_version: version, + completed: { + ...(state.completed || {}), + [deploymentId]: { version, desired_state: desiredState, status: "healthy", completed_at: new Date().toISOString() }, + }, + }); + return { accepted: true, duplicate: false }; + } catch (rolloutError) { + if (previousVersion && desiredState === "running") { + try { + docker(["pull", "engine"], previousVersion); + docker(["up", "-d", "engine"], previousVersion); + const restored = await waitForEngineHealth(previousVersion); + if (restored) { + await reportResult(deploymentId, { + status: "rolled_back", + observed_state: "running", + version: previousVersion, + evidence: { ...baseEvidence, rollback_version: previousVersion, rollback_health_check: true, health_check: true }, + message: "The requested release failed its health check; the previous known-good release was restored.", + }); + await saveState({ + ...state, + last_known_good_version: previousVersion, + completed: { + ...(state.completed || {}), + [deploymentId]: { version: previousVersion, desired_state: "running", status: "rolled_back", completed_at: new Date().toISOString() }, + }, + }); + return { accepted: true, rolled_back: true }; + } + } catch { + // Fall through to a failed result; do not claim rollback without a passing health check. + } + } + await reportResult(deploymentId, { + status: "failed", + observed_state: "unknown", + version, + message: String(rolloutError.message || rolloutError).slice(0, 900), + evidence: { ...baseEvidence, health_check: false, rollback_attempted: Boolean(previousVersion) }, + }); + await saveState({ + ...state, + completed: { + ...(state.completed || {}), + [deploymentId]: { version, desired_state: desiredState, status: "failed", completed_at: new Date().toISOString() }, + }, + }); + return { accepted: true, failed: true }; + } + } finally { + inFlight.delete(deploymentId); + } +} + +const server = createServer((req, res) => { + const respond = (status, body) => { + res.writeHead(status, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" }); + res.end(JSON.stringify(body)); + }; + if (req.method !== "POST" || req.url !== "/webhooks/agata/deployments") { + respond(404, { error: "not_found" }); + return; + } + if (!String(req.headers["content-type"] || "").toLowerCase().startsWith("application/json")) { + respond(415, { error: "application_json_required" }); + return; + } + const chunks = []; + let size = 0; + let tooLarge = false; + req.on("data", (chunk) => { + size += chunk.length; + if (size > maxBody) { tooLarge = true; chunks.length = 0; } + else if (!tooLarge) chunks.push(chunk); + }); + req.on("end", async () => { + if (tooLarge) return respond(413, { error: "payload_too_large" }); + const rawBody = Buffer.concat(chunks); + if (!verifySignature(rawBody, req.headers["x-agata-signature"])) { + return respond(401, { error: "invalid_signature" }); + } + const deliveryId = String(req.headers["x-agata-delivery-id"] || ""); + if (!deliveryId || deliveryId.length > 200) return respond(400, { error: "invalid_delivery_id" }); + let event; + try { event = JSON.parse(rawBody.toString("utf8")); } + catch { return respond(400, { error: "invalid_json" }); } + if (event.type !== "deployment.created") return respond(204, {}); + try { + const result = await executeDeployment(event, deliveryId); + return respond(202, result); + } catch (error) { + console.error(JSON.stringify({ error: String(error.message || error).slice(0, 1000), deliveryId })); + return respond(503, { error: "deployment_execution_deferred" }); + } + }); + req.on("error", () => { if (!res.headersSent) respond(400, { error: "request_read_failed" }); }); +}); + +server.requestTimeout = 180_000; +server.headersTimeout = 185_000; +server.listen(port, () => console.log(`Agata Proxima deployment executor listening on port ${port}`)); From ad8adc7a1e24a2b3363e4ab992c97fbeb29e32ce Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:44:22 -0800 Subject: [PATCH 017/108] docs(engine): document customer deployment executor installation --- docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md | 90 ++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md diff --git a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md new file mode 100644 index 0000000..830ac74 --- /dev/null +++ b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md @@ -0,0 +1,90 @@ +# Customer-managed Engine deployment executor + +This reference executor connects Agata Proxima's signed deployment event to a customer's Docker Compose installation of Proxima Engine. It runs inside the customer's infrastructure and has permission to control the customer's Engine container. + +## What it does + +1. Receives deployment.created over HTTPS. +2. Verifies the exact raw-body HMAC-SHA256 signature and five-minute timestamp window. +3. Rejects events for a different node or a release version outside the customer's explicit allowlist. +4. Sends the applying callback with a stable executor and execution ID. +5. Pulls the requested versioned Engine image and applies the requested running or stopped state using fixed Docker Compose arguments. It never executes command text from a webhook. +6. Checks the real container state. A running Engine must pass its local /healthz endpoint. +7. Sends healthy only after the check passes. If a running rollout fails and a prior known-good release is recorded, it restores that release and reports rolled_back. +8. Persists completed deployment IDs locally. It can recover from a lost local completion marker by inspecting the authoritative Control Plane deployment result before retrying side effects. + +The draining state is intentionally rejected because a safe drain operation requires an application-specific connection-drain controller. Do not select that state unless you have configured and tested such a controller. + +## Prerequisites + +- Linux or Windows host with Docker Engine and Docker Compose v2. +- Node.js 22 or newer. +- The Engine already installed with compose.customer.yaml. +- A published Engine container image for each release version you intend to deploy. +- One organization-scoped API key with permission to write deployment results. +- Exactly one enabled HTTPS webhook subscribed to deployment.created for the organization. + +The Engine image is published by the tagged-release workflow. For a tag such as v0.1.0, the workflow publishes both 0.1.0 and v0.1.0 plus latest. Make the ghcr.io/proxima-engine/proxima-engine package public before asking unauthenticated customer hosts to pull it; otherwise configure Docker registry authentication on the customer host. + +## Install + +Download this repository's ZIP and extract it on the same customer host that runs the Engine Compose project. Create a protected environment file for the executor. Do not commit it: + +```dotenv +AGATA_API_BASE_URL=https://api.agataproxima.com +AGATA_API_KEY=aga_ +AGATA_WEBHOOK_SIGNING_SECRET= +PROXIMA_NODE_ID= +PROXIMA_DEPLOYMENT_EXECUTOR_ID=customer-prod-engine-01 +PROXIMA_ENGINE_ALLOWLIST=0.1.0,0.1.1 +PROXIMA_DEPLOYMENT_PROJECT_DIR=/opt/proxima +PROXIMA_DEPLOYMENT_COMPOSE_FILE=compose.customer.yaml +PROXIMA_DEPLOYMENT_STATE_DIR=/var/lib/agata-deployment-executor +PORT=8789 +``` + +The allowlist must contain only Engine releases the customer has reviewed and approved. Incoming webhook values are never added to it. + +Run the service behind a customer-managed HTTPS reverse proxy at: + +POST https:///webhooks/agata/deployments + +For a smoke test: + +```bash +set -a +. /etc/agata-deployment-executor.env +set +a +node ./deployment-executor/receiver.mjs +``` + +The reverse proxy must preserve the raw request body and forward X-Agata-Signature, X-Agata-Delivery-Id, X-Agata-Event, and X-Agata-Event-Id. + +## Configure Agata Proxima + +1. Open Developer → API keys and create a dedicated write-capable key for the executor. +2. Open Developer → Webhooks and create one HTTPS endpoint using the route above. +3. Subscribe to exactly deployment.created. Do not use * for this executor. +4. Copy the signing secret shown once and store it as AGATA_WEBHOOK_SIGNING_SECRET. +5. Confirm there is only one enabled webhook subscribed to deployment.created for the organization. +6. Confirm the target node is healthy in Node Management. +7. Create a staging deployment using a version in the allowlist. + +## Supported states and rollback + +- running: pull the versioned image, start the Engine and require /healthz to pass. +- stopped: pull the versioned image, stop the Engine and verify the service is no longer running. +- draining: rejected. Add an application-specific drain controller before enabling this state. + +A failed running rollout attempts to restore the previous version only when that version is recorded as known-good and remains in the allowlist. The executor reports rolled_back only after the restored Engine passes its health check. If rollback cannot be verified, it reports failed. + +## Security and operations + +- This process can control Docker and therefore has high privilege on its host. Run it as a dedicated OS identity and restrict access to its environment file. +- Access to the Docker socket or Docker group is effectively host-administrator access. Do not expose the Docker socket over TCP. +- Keep the API key, webhook secret, and node identity separate. Rotate the API key and webhook secret independently. +- Keep the executor state directory persistent and protected. It stores deployment IDs and release metadata, not API keys or webhook secrets. +- The webhook receiver returns 503 when execution or callback delivery cannot be completed, allowing the Control Plane delivery queue to retry. +- Test invalid signatures, duplicate delivery, image pull failure, health-check failure, rollback success and rollback failure in staging before production use. + +Agata Proxima records deployment intent and verified execution results. The customer remains in control of the Docker host, database, release approvals and infrastructure credentials. From be81bf196add7fe08099e1d452d2d9e686d33a89 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:44:30 -0800 Subject: [PATCH 018/108] docs(engine): link customer deployment executor --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 39e493f..4078bf0 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ LinkedIn

-**Start:** [What is Agata Proxima?](#what-is-agata-proxima) · [Customer Docker install](docs/NODE_ENROLLMENT.md#2-one-command-docker-installation-hosted-cloud) · [Launch evaluation](docs/LAUNCH.md) · [Quickstart](docs/QUICKSTART.md) · [Local developer mode](docs/LOCAL_DEVELOPMENT.md) · [Engine CLI](docs/CLI.md) · [Documentation index](docs/README.md) +**Start:** [What is Agata Proxima?](#what-is-agata-proxima) · [Customer Docker install](docs/NODE_ENROLLMENT.md#2-one-command-docker-installation-hosted-cloud) · [Deployment executor](docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md) · [Launch evaluation](docs/LAUNCH.md) · [Quickstart](docs/QUICKSTART.md) · [Local developer mode](docs/LOCAL_DEVELOPMENT.md) · [Engine CLI](docs/CLI.md) · [Documentation index](docs/README.md) **Technical guides:** [Architecture](docs/ARCHITECTURE.md) · [Tenant isolation](docs/TENANT_ISOLATION.md) · [Driver integrations](docs/INTEGRATIONS.md) · [API and metrics](docs/API.md) · [Security](docs/SECURITY.md) · [Deployment](docs/DEPLOYMENT.md) · [Verification](docs/VERIFICATION.md) @@ -129,6 +129,7 @@ See [Tenant isolation](docs/TENANT_ISOLATION.md), [Verification](docs/VERIFICATI | [Driver integrations](docs/INTEGRATIONS.md) | psql, Node.js, Python, and Rust connection examples | | [API reference](docs/API.md) | PostgreSQL endpoint, health/status routes, and Prometheus metrics | | [Node enrollment](docs/NODE_ENROLLMENT.md) | Customer Docker installation, one-time enrollment, signed heartbeat, and identity persistence | +| [Customer deployment executor](docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md) | Install and operate the customer-managed, signed Docker Compose deployment executor | | [Deployment](docs/DEPLOYMENT.md) | Container build, TLS, networking, persistence, and deployment configuration | | [Operations](docs/OPERATIONS.md) | Health checks, logs, monitoring, recovery, and troubleshooting | | [Engineering case study](docs/ENGINEERING_CASE_STUDY.md) | Executed tenant-isolation, RLS, PgBouncer, concurrency, authentication, and recovery results | From ec15afd280fdb0fba48550eaf5589b86c71e63f3 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:44:32 -0800 Subject: [PATCH 019/108] ci(engine): syntax-check customer deployment executor --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3b20f8..c208b2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,8 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Test public Engine CLI run: python3 -m unittest discover -s tests_cli -v + - name: Validate customer deployment executor syntax + run: node --check deployment-executor/receiver.mjs - name: Run engine tests run: cargo test -p proxima-engine - name: Build release engine From d3175b704da6bbcad44b8856050c4da422c8d607 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:44:34 -0800 Subject: [PATCH 020/108] docs(engine): cross-link customer deployment executor --- docs/NODE_ENROLLMENT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/NODE_ENROLLMENT.md b/docs/NODE_ENROLLMENT.md index 99d1e52..bc3a7fb 100644 --- a/docs/NODE_ENROLLMENT.md +++ b/docs/NODE_ENROLLMENT.md @@ -145,4 +145,4 @@ These credentials serve different trust boundaries. Keep each in its designated - **Identity file error:** check the path, JSON format, and read/write permissions. - **Engine exits before listening:** inspect the startup error and verify all supplied enrollment values. -See [Configuration](CONFIGURATION.md), [Operations](OPERATIONS.md), and [Cloud control-plane boundary](CLOUD_CONTROL_PLANE.md). +See [Configuration](CONFIGURATION.md), [Operations](OPERATIONS.md), [Customer deployment executor](CUSTOMER_DEPLOYMENT_EXECUTOR.md), and [Cloud control-plane boundary](CLOUD_CONTROL_PLANE.md). From 9d1b0be167d9524f6a3f59d7f22106c605135a9f Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:45:54 -0800 Subject: [PATCH 021/108] feat(engine): read one-time enrollment token from Compose secret --- crates/proxima-engine/src/main.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/proxima-engine/src/main.rs b/crates/proxima-engine/src/main.rs index 1f13e6b..f5a4899 100644 --- a/crates/proxima-engine/src/main.rs +++ b/crates/proxima-engine/src/main.rs @@ -246,7 +246,13 @@ async fn bootstrap_node_identity() -> Result<(), Box Result<(), Box Date: Tue, 10 Nov 2026 01:46:21 -0800 Subject: [PATCH 022/108] feat(engine): support tenant signing key Compose secret files --- crates/proxima-engine/src/config.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/proxima-engine/src/config.rs b/crates/proxima-engine/src/config.rs index 70713ed..a34e0d4 100644 --- a/crates/proxima-engine/src/config.rs +++ b/crates/proxima-engine/src/config.rs @@ -78,16 +78,20 @@ impl Config { )); } - let tenant_signing_key = match env::var("PROXIMA_TENANT_SIGNING_KEY") { - Ok(value) if value.trim().is_empty() => None, - Ok(value) if value.len() < 32 => { + let tenant_signing_key_value = match env::var("PROXIMA_TENANT_SIGNING_KEY_FILE") { + Ok(path) => Some(std::fs::read_to_string(path)?.trim().to_owned()), + Err(_) => env::var("PROXIMA_TENANT_SIGNING_KEY").ok(), + }; + let tenant_signing_key = match tenant_signing_key_value { + Some(value) if value.trim().is_empty() => None, + Some(value) if value.as_bytes().len() < 32 => { return Err(io::Error::new( io::ErrorKind::InvalidInput, - "PROXIMA_TENANT_SIGNING_KEY must be at least 32 bytes", + "PROXIMA_TENANT_SIGNING_KEY or its secret file must contain at least 32 bytes", )) } - Ok(value) => Some(value), - Err(_) => None, + Some(value) => Some(value), + None => None, }; let upstream_connect_timeout_ms = env::var("PROXIMA_UPSTREAM_CONNECT_TIMEOUT_MS") @@ -241,6 +245,7 @@ mod tests { "PROXIMA_LISTEN_ADDR", "PROXIMA_UPSTREAM_ADDR", "PROXIMA_TENANT_SIGNING_KEY", + "PROXIMA_TENANT_SIGNING_KEY_FILE", "PROXIMA_UPSTREAM_CONNECT_TIMEOUT_MS", "PROXIMA_MAX_CONNECTIONS", "PROXIMA_TLS_CERT_FILE", From 51aa592588719afa1432048a5c3569ff1e09cb36 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:46:45 -0800 Subject: [PATCH 023/108] feat(engine): mount node and tenant credentials as Compose secrets --- compose.customer.yaml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/compose.customer.yaml b/compose.customer.yaml index 0f227d4..7ede85d 100644 --- a/compose.customer.yaml +++ b/compose.customer.yaml @@ -5,19 +5,22 @@ services: context: . dockerfile: Dockerfile restart: unless-stopped - env_file: - - .env environment: + PROXIMA_NODE_ID: ${PROXIMA_NODE_ID:?Set PROXIMA_NODE_ID in .env} PROXIMA_NODE_IDENTITY_PATH: /var/lib/proxima/node-identity.json - PROXIMA_LISTEN_ADDR: ${PROXIMA_LISTEN_ADDR:-0.0.0.0:6432} - PROXIMA_UPSTREAM_ADDR: ${PROXIMA_UPSTREAM_ADDR:?Set PROXIMA_UPSTREAM_ADDR in .env} - PROXIMA_TENANT_SIGNING_KEY: ${PROXIMA_TENANT_SIGNING_KEY:?Set PROXIMA_TENANT_SIGNING_KEY in .env} PROXIMA_NODE_ENROLLMENT_URL: ${PROXIMA_NODE_ENROLLMENT_URL:-https://api.agataproxima.com/api/v1/nodes/enroll} PROXIMA_NODE_HEARTBEAT_URL: ${PROXIMA_NODE_HEARTBEAT_URL:-https://api.agataproxima.com/api/v1/nodes/heartbeat} + PROXIMA_NODE_ENROLLMENT_TOKEN_FILE: /run/secrets/node_enrollment_token + PROXIMA_TENANT_SIGNING_KEY_FILE: /run/secrets/tenant_signing_key + PROXIMA_LISTEN_ADDR: ${PROXIMA_LISTEN_ADDR:-0.0.0.0:6432} + PROXIMA_UPSTREAM_ADDR: ${PROXIMA_UPSTREAM_ADDR:?Set PROXIMA_UPSTREAM_ADDR in .env} PROXIMA_DASHBOARD_ENABLED: "false" PROXIMA_METRICS_LISTEN_ADDR: 127.0.0.1:9090 ports: - "${PROXIMA_BIND_ADDRESS:-127.0.0.1}:6432:6432" + secrets: + - node_enrollment_token + - tenant_signing_key volumes: - proxima-node-data:/var/lib/proxima healthcheck: @@ -31,5 +34,16 @@ services: read_only: true tmpfs: - /tmp:size=16m,noexec,nosuid + +secrets: + node_enrollment_token: + file: ./secrets/node_enrollment_token + tenant_signing_key: + file: ./secrets/tenant_signing_key + +networks: + default: + name: ${PROXIMA_DOCKER_NETWORK:-proxima-engine-network} + volumes: proxima-node-data: From 5a0fbd17877e60b3c6f3027aecdbbc07ab48d208 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:47:31 -0800 Subject: [PATCH 024/108] docs(engine): align customer setup with Compose secret downloads --- docs/NODE_ENROLLMENT.md | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/docs/NODE_ENROLLMENT.md b/docs/NODE_ENROLLMENT.md index bc3a7fb..6c0254b 100644 --- a/docs/NODE_ENROLLMENT.md +++ b/docs/NODE_ENROLLMENT.md @@ -41,39 +41,26 @@ Use HTTPS for remote control-plane hosts. Loopback HTTP is reserved for local de ## 2. One-command Docker installation (hosted Cloud) -The hosted customer path uses the published Engine container. Docker and Docker Compose are required on the customer-managed host. The customer keeps PostgreSQL and its tenant signing key under their own control; Agata Proxima does not create or take ownership of the customer's database. - -1. Register a node in Agata Proxima Cloud and copy its one-time enrollment token. -2. Download `compose.customer.yaml` from this repository and save it as `compose.yaml` in a protected directory. -3. Create a `.env` file beside it with the values below. Do not commit this file. - -```dotenv -PROXIMA_NODE_ID= -PROXIMA_NODE_ENROLLMENT_URL=https://api.agataproxima.com/api/v1/nodes/enroll -PROXIMA_NODE_ENROLLMENT_TOKEN= -PROXIMA_UPSTREAM_ADDR=:5432 -PROXIMA_TENANT_SIGNING_KEY= -PROXIMA_LISTEN_ADDR=0.0.0.0:6432 -``` +This is the recommended customer path. Docker Engine and Docker Compose v2 are required on the customer's host. Agata Proxima does not provision, own, or receive the customer's PostgreSQL credentials. -Generate a tenant signing key locally (never reuse the node enrollment token as this key): +1. Register a node in Agata Proxima Cloud and download the three generated files from the registration result: .env, node_enrollment_token, and tenant_signing_key. +2. Download the public Engine repository ZIP and extract it on the customer host. +3. Place .env and compose.customer.yaml in the repository root. Create a secrets directory beside them and save the two secret downloads as secrets/node_enrollment_token and secrets/tenant_signing_key. +4. Set the upstream address in .env to the PostgreSQL host reachable from the Engine container. The tenant signing key must be the same secret the trusted application backend uses to sign tenant context. Never reuse the one-time enrollment token as the tenant signing key. +5. Start the Engine: ```bash -openssl rand -hex 32 +docker compose -f compose.customer.yaml up --build -d +docker compose -f compose.customer.yaml logs -f engine ``` -4. Start the Engine: - -```bash -docker compose -f compose.yaml up -d -docker compose -f compose.yaml logs -f engine -``` +The Compose file mounts the enrollment token and tenant signing key as read-only secret files. Neither secret is passed as a container environment variable. The Engine reads the enrollment token file only during first enrollment, persists its node identity in the named Docker volume, and sends signed heartbeats every 30 seconds. No heartbeat cron job, custom webhook receiver, or customer-written enrollment code is required. -5. Confirm the container is healthy and the node appears as healthy in Fleet. The Engine redeems the one-time token, persists its node identity in the named Docker volume, and sends signed heartbeats every 30 seconds. No heartbeat cron job, custom webhook receiver, or customer-written enrollment code is required. +When Fleet shows the node as healthy, replace the contents of secrets/node_enrollment_token with the harmless text enrollment-complete. The persisted node identity is used on subsequent starts, so the consumed one-time token is no longer needed. Keep secrets/tenant_signing_key protected and persistent. -For PowerShell, create the same `.env` file with a text editor and run the same Docker Compose commands. Protect the directory and volume because the persisted identity contains the node signing credential. +For PowerShell, use the same files and Docker Compose commands. The host directory containing .env and secrets must be accessible only to the customer operators who manage the Engine. -The container listens on port 6432. Keep that port private to the application network or restrict it with the customer's firewall. Configure the application to connect to the Engine, not directly to PostgreSQL. The upstream address must be reachable from inside the container; `localhost` means the container itself, not the customer's host. +The Engine listens on port 6432. By default, the host port is bound to 127.0.0.1. If the application runs in another container, attach it to the shared proxima-engine-network and connect to engine:6432. Do not expose port 6432 publicly without a customer-controlled firewall rule. The upstream PostgreSQL address must be reachable from inside the Engine container; localhost means the container itself, not the host. ## 3. Configure the Engine — PowerShell From 669332532d609ef1cdef35c802c4e7917458997b Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:51:26 -0800 Subject: [PATCH 025/108] docs(engine): document tenant signing secret file configuration --- docs/CONFIGURATION.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 609adfe..43d6c7b 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -8,7 +8,8 @@ The Engine reads settings from its process environment. Defaults and validation | --- | --- | --- | | `PROXIMA_LISTEN_ADDR` | `127.0.0.1:6432` | PostgreSQL client listener | | `PROXIMA_UPSTREAM_ADDR` | `127.0.0.1:5432` | PostgreSQL server address reachable by the Engine | -| `PROXIMA_TENANT_SIGNING_KEY` | Unset | HMAC key for tenant tokens; at least 32 bytes. If absent, tenant-token enforcement is not enabled | +| `PROXIMA_TENANT_SIGNING_KEY` | Unset | HMAC key for tenant tokens; at least 32 bytes. Prefer a secret file for customer deployments | +| `PROXIMA_TENANT_SIGNING_KEY_FILE` | Unset | Path to a mounted secret file containing the tenant HMAC key; takes precedence over the direct environment variable | | `PROXIMA_TENANT_ROLE_PREFIX` | `proxima_tenant_` | Prefix for tenant PostgreSQL roles; ASCII letters, digits, and underscores only | | `PROXIMA_UPSTREAM_CONNECT_TIMEOUT_MS` | `10000` | Upstream connect timeout; must be positive | | `PROXIMA_MAX_CONNECTIONS` | `1024` | Maximum simultaneous client connections; must be positive | From e078c9a4db60a6e092b9cfdd7da98f2a1c4466c6 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:51:28 -0800 Subject: [PATCH 026/108] docs(engine): recommend mounted tenant signing key secret files --- docs/SECURITY.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 6ef4750..bf9bdaa 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -6,7 +6,7 @@ This guide describes the Engine controls, their activation settings, and the ope ## Implemented security controls -- **Signed tenant context:** when `PROXIMA_TENANT_SIGNING_KEY` is configured, the Engine requires a valid, unexpired HMAC-SHA256 tenant token during PostgreSQL startup. Missing, malformed, expired, and incorrectly signed tokens are rejected. +- **Signed tenant context:** when `PROXIMA_TENANT_SIGNING_KEY` or `PROXIMA_TENANT_SIGNING_KEY_FILE` is configured, the Engine requires a valid, unexpired HMAC-SHA256 tenant token during PostgreSQL startup. Prefer the file setting with a mounted secret in customer deployments. Missing, malformed, expired, and incorrectly signed tokens are rejected. - **Tenant-specific role routing:** the verified tenant ID is mapped through `PROXIMA_TENANT_ROLE_PREFIX` to the corresponding PostgreSQL login role. Each accepted Engine client session has its own upstream PostgreSQL session. - **PostgreSQL authorization:** tenant roles operate under PostgreSQL grants and RLS policies. Use non-superuser roles without `BYPASSRLS`, and apply `FORCE ROW LEVEL SECURITY` where the schema requires it. - **Client and upstream TLS:** configure the client certificate/key pair and set `PROXIMA_TLS_REQUIRE_CLIENT=true` to reject plaintext client connections. Configure upstream `verify-full` with a trusted CA and expected server name. @@ -27,7 +27,7 @@ The Engine keeps each accepted client connection bound to its own upstream sessi ## Required deployment configuration - [ ] Pin and review the exact source commit or container image digest. -- [ ] Set `PROXIMA_TENANT_SIGNING_KEY` and verify that tenant-token enforcement is active in the Engine status output. +- [ ] Mount the tenant signing key as a secret file and set `PROXIMA_TENANT_SIGNING_KEY_FILE`; verify that tenant-token enforcement is active in the Engine status output. - [ ] Issue short-lived tenant tokens only from a trusted backend after authenticating the user and authorizing tenant membership. - [ ] Use non-superuser PostgreSQL tenant roles without `BYPASSRLS`. - [ ] Apply and test RLS policies for every tenant-scoped table and relevant read/write operation. From 26bd0296d7eae3159566bc96a869d4a1fb101e97 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:52:05 -0800 Subject: [PATCH 027/108] ci(engine): validate customer Compose secrets and required configuration --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c208b2d..9988bc6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,12 @@ jobs: run: python3 -m unittest discover -s tests_cli -v - name: Validate customer deployment executor syntax run: node --check deployment-executor/receiver.mjs + - name: Validate customer Compose installation contract + run: | + mkdir -p secrets + printf 'test-enrollment-token' > secrets/node_enrollment_token + printf '0123456789abcdef0123456789abcdef' > secrets/tenant_signing_key + PROXIMA_NODE_ID=00000000-0000-4000-8000-000000000001 PROXIMA_UPSTREAM_ADDR=127.0.0.1:5432 docker compose -f compose.customer.yaml config --quiet - name: Run engine tests run: cargo test -p proxima-engine - name: Build release engine From 4534f059a5910f41fb2b20c356ccc2e73e9fa05d Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:56:55 -0800 Subject: [PATCH 028/108] fix(engine): align executor timeout with Control Plane rollout delivery --- deployment-executor/receiver.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment-executor/receiver.mjs b/deployment-executor/receiver.mjs index 6aa4b36..aadb8b2 100644 --- a/deployment-executor/receiver.mjs +++ b/deployment-executor/receiver.mjs @@ -300,6 +300,6 @@ const server = createServer((req, res) => { req.on("error", () => { if (!res.headersSent) respond(400, { error: "request_read_failed" }); }); }); -server.requestTimeout = 180_000; -server.headersTimeout = 185_000; +server.requestTimeout = 360_000; +server.headersTimeout = 365_000; server.listen(port, () => console.log(`Agata Proxima deployment executor listening on port ${port}`)); From 17a119c6074a54d4a19562b91077ec25f4e446a0 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:57:08 -0800 Subject: [PATCH 029/108] docs(engine): document deployment webhook response timeout --- docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md index 830ac74..b751ded 100644 --- a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md +++ b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md @@ -58,7 +58,7 @@ set +a node ./deployment-executor/receiver.mjs ``` -The reverse proxy must preserve the raw request body and forward X-Agata-Signature, X-Agata-Delivery-Id, X-Agata-Event, and X-Agata-Event-Id. +The reverse proxy must preserve the raw request body and forward X-Agata-Signature, X-Agata-Delivery-Id, X-Agata-Event, and X-Agata-Event-Id. Keep its upstream request timeout at or above six minutes: Agata Proxima allows deployment.created delivery up to six minutes for image pulls, Engine restarts and runtime health checks. The executor returns only after the Control Plane has accepted the execution result. ## Configure Agata Proxima From e3f17dac472c88ed68b62be9ffb5c7120506a332 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:57:57 -0800 Subject: [PATCH 030/108] feat(engine): support one signed executor for multiple customer nodes --- deployment-executor/receiver.mjs | 87 ++++++++++++++++++++------------ 1 file changed, 54 insertions(+), 33 deletions(-) diff --git a/deployment-executor/receiver.mjs b/deployment-executor/receiver.mjs index aadb8b2..26957a2 100644 --- a/deployment-executor/receiver.mjs +++ b/deployment-executor/receiver.mjs @@ -9,32 +9,51 @@ const required = [ "AGATA_API_BASE_URL", "AGATA_API_KEY", "AGATA_WEBHOOK_SIGNING_SECRET", - "PROXIMA_NODE_ID", "PROXIMA_DEPLOYMENT_EXECUTOR_ID", - "PROXIMA_ENGINE_ALLOWLIST", ]; for (const key of required) { if (!process.env[key]?.trim()) throw new Error(`Missing required environment variable: ${key}`); } -const apiBase = process.env.AGATA_API_BASE_URL.replace(/\/+$/, ""); +const apiBase = process.env.AGATA_API_BASE_URL.replace(/\\/+$/, ""); const apiKey = process.env.AGATA_API_KEY; const webhookSecret = process.env.AGATA_WEBHOOK_SIGNING_SECRET; -const nodeId = process.env.PROXIMA_NODE_ID; const executorId = process.env.PROXIMA_DEPLOYMENT_EXECUTOR_ID; -const allowedVersions = new Set(process.env.PROXIMA_ENGINE_ALLOWLIST.split(",").map((v) => v.trim()).filter(Boolean)); -const projectDir = path.resolve(process.env.PROXIMA_DEPLOYMENT_PROJECT_DIR || process.cwd()); -const composeFile = path.resolve(projectDir, process.env.PROXIMA_DEPLOYMENT_COMPOSE_FILE || "compose.customer.yaml"); -const stateDir = path.resolve(process.env.PROXIMA_DEPLOYMENT_STATE_DIR || path.join(projectDir, ".agata-deployments")); -const stateFile = path.join(stateDir, "completed.json"); const port = Number(process.env.PORT || 8789); const maxBody = 1024 * 1024; const inFlight = new Set(); +const defaultStateDir = path.resolve(process.env.PROXIMA_DEPLOYMENT_STATE_DIR || path.join(process.cwd(), ".agata-deployments")); +const targets = new Map(); -if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error("PORT must be a valid TCP port."); -if (allowedVersions.size === 0 || [...allowedVersions].some((v) => !/^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$/.test(v))) { - throw new Error("PROXIMA_ENGINE_ALLOWLIST must contain explicit, valid Engine release tags."); +if (process.env.PROXIMA_DEPLOYMENT_TARGETS_FILE?.trim()) { + const targetFile = path.resolve(process.env.PROXIMA_DEPLOYMENT_TARGETS_FILE); + const config = JSON.parse(await readFile(targetFile, "utf8")); + for (const [nodeId, value] of Object.entries(config)) { + if (!/^[0-9a-f-]{36}$/i.test(nodeId)) throw new Error(`Invalid node UUID in deployment target file: ${nodeId}`); + const projectDir = path.resolve(String(value.project_dir || "")); + const composePath = path.resolve(projectDir, String(value.compose_file || "compose.customer.yaml")); + const stateDir = path.resolve(String(value.state_dir || path.join(defaultStateDir, nodeId))); + const allowedVersions = new Set(Array.isArray(value.allowed_versions) ? value.allowed_versions.map(v => String(v).trim()).filter(Boolean) : []); + if (!value.project_dir || allowedVersions.size === 0 || [...allowedVersions].some(v => !/^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$/.test(v))) { + throw new Error(`Target ${nodeId} requires project_dir and an explicit allowed_versions array.`); + } + targets.set(nodeId, { projectDir, composeFile: composePath, stateDir, stateFile: path.join(stateDir, "completed.json"), allowedVersions }); + } +} else { + const nodeId = process.env.PROXIMA_NODE_ID; + const versions = (process.env.PROXIMA_ENGINE_ALLOWLIST || "").split(",").map(v => v.trim()).filter(Boolean); + if (!nodeId || !/^[0-9a-f-]{36}$/i.test(nodeId) || !versions.length) { + throw new Error("Set PROXIMA_DEPLOYMENT_TARGETS_FILE or configure PROXIMA_NODE_ID and PROXIMA_ENGINE_ALLOWLIST for a single-node executor."); + } + const projectDir = path.resolve(process.env.PROXIMA_DEPLOYMENT_PROJECT_DIR || process.cwd()); + const composeFile = path.resolve(projectDir, process.env.PROXIMA_DEPLOYMENT_COMPOSE_FILE || "compose.customer.yaml"); + const stateDir = path.resolve(process.env.PROXIMA_DEPLOYMENT_STATE_DIR || path.join(projectDir, ".agata-deployments")); + const allowedVersions = new Set(versions); + if ([...allowedVersions].some(v => !/^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$/.test(v))) throw new Error("PROXIMA_ENGINE_ALLOWLIST contains an invalid Engine release version."); + targets.set(nodeId, { projectDir, composeFile, stateDir, stateFile: path.join(stateDir, "completed.json"), allowedVersions }); } +if (targets.size === 0) throw new Error("At least one deployment target must be configured."); +if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error("PORT must be a valid TCP port."); if (new URL(apiBase).protocol !== "https:" && !["localhost", "127.0.0.1"].includes(new URL(apiBase).hostname)) { throw new Error("AGATA_API_BASE_URL must use HTTPS outside local development."); } @@ -53,21 +72,21 @@ function verifySignature(rawBody, header) { return actual.length === expected.length && timingSafeEqual(actual, expected); } -async function readState() { +async function readState(stateFile) { try { return JSON.parse(await readFile(stateFile, "utf8")); } catch (error) { if (error.code === "ENOENT") return { completed: {} }; throw error; } } -async function saveState(state) { - await mkdir(stateDir, { recursive: true, mode: 0o700 }); +async function saveState(stateFile, state) { + await mkdir(path.dirname(stateFile), { recursive: true, mode: 0o700 }); const temporary = `${stateFile}.tmp`; await writeFile(temporary, JSON.stringify(state, null, 2), { mode: 0o600 }); await rename(temporary, stateFile); } -function docker(args, version) { - const result = spawnSync("docker", ["compose", "-f", composeFile, ...args], { - cwd: projectDir, +function docker(target, args, version) { + const result = spawnSync("docker", ["compose", "-f", target.composeFile, ...args], { + cwd: target.projectDir, env: { ...process.env, PROXIMA_ENGINE_VERSION: version }, encoding: "utf8", timeout: 120_000, @@ -84,10 +103,10 @@ function docker(args, version) { const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); -async function waitForEngineHealth(version) { +async function waitForEngineHealth(target, version) { for (let attempt = 0; attempt < 12; attempt += 1) { try { - docker(["exec", "-T", "engine", "curl", "-fsS", "http://127.0.0.1:9090/healthz"], version); + docker(target, ["exec", "-T", "engine", "curl", "-fsS", "http://127.0.0.1:9090/healthz"], version); return true; } catch { await sleep(5000); @@ -133,11 +152,13 @@ async function executeDeployment(event, deliveryId) { const desiredState = String(data.desired_state || ""); if (!/^[0-9a-f-]{36}$/i.test(deploymentId)) throw new Error("Deployment event has an invalid resource ID."); - if (targetNode !== nodeId) throw new Error("Deployment event targets a different Engine node."); - if (!allowedVersions.has(version)) throw new Error("Requested Engine version is not in PROXIMA_ENGINE_ALLOWLIST."); + const target = targets.get(targetNode); + if (!target) throw new Error("No customer deployment target is configured for this node ID."); + if (!target.allowedVersions.has(version)) throw new Error("Requested Engine version is not in this node target allowlist."); if (!["running", "stopped"].includes(desiredState)) throw new Error("This reference executor supports running and stopped states; draining requires a configured drain controller."); - const state = await readState(); + const stateFile = target.stateFile; + const state = await readState(stateFile); if (state.completed[deploymentId]) return { accepted: true, duplicate: true }; if (inFlight.has(deploymentId)) throw new Error("This deployment is already executing; retry the webhook delivery."); inFlight.add(deploymentId); @@ -153,7 +174,7 @@ async function executeDeployment(event, deliveryId) { if (evidence.executor !== executorId || evidence.execution_id !== executionId) { throw new Error("Deployment already has a terminal result from a different executor."); } - await saveState({ + await saveState(stateFile, { ...state, last_known_good_version: remote.applied_version || state.last_known_good_version, completed: { @@ -175,19 +196,19 @@ async function executeDeployment(event, deliveryId) { evidence: baseEvidence, }); - const previousVersion = state.last_known_good_version && allowedVersions.has(state.last_known_good_version) + const previousVersion = state.last_known_good_version && target.allowedVersions.has(state.last_known_good_version) ? state.last_known_good_version : null; try { - docker(["pull", "engine"], version); + docker(target, ["pull", "engine"], version); if (desiredState === "running") { - docker(["up", "-d", "engine"], version); - const healthy = await waitForEngineHealth(version); + docker(target, ["up", "-d", "engine"], version); + const healthy = await waitForEngineHealth(target, version); if (!healthy) throw new Error("Engine did not pass its local /healthz check after rollout."); } else { - docker(["stop", "engine"], version); - const runningServices = docker(["ps", "--status", "running", "--services"], version) + docker(target, ["stop", "engine"], version); + const runningServices = docker(target, ["ps", "--status", "running", "--services"], version) .split(/\r?\n/).filter(Boolean); if (runningServices.includes("engine")) throw new Error("Engine container is still running after the stop request."); } @@ -210,9 +231,9 @@ async function executeDeployment(event, deliveryId) { } catch (rolloutError) { if (previousVersion && desiredState === "running") { try { - docker(["pull", "engine"], previousVersion); - docker(["up", "-d", "engine"], previousVersion); - const restored = await waitForEngineHealth(previousVersion); + docker(target, ["pull", "engine"], previousVersion); + docker(target, ["up", "-d", "engine"], previousVersion); + const restored = await waitForEngineHealth(target, previousVersion); if (restored) { await reportResult(deploymentId, { status: "rolled_back", From 0be9dc9e9f65c66557989b097988d27db944964c Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:58:15 -0800 Subject: [PATCH 031/108] fix(engine): persist per-node deployment executor state correctly --- deployment-executor/receiver.mjs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deployment-executor/receiver.mjs b/deployment-executor/receiver.mjs index 26957a2..d99e0e0 100644 --- a/deployment-executor/receiver.mjs +++ b/deployment-executor/receiver.mjs @@ -15,7 +15,7 @@ for (const key of required) { if (!process.env[key]?.trim()) throw new Error(`Missing required environment variable: ${key}`); } -const apiBase = process.env.AGATA_API_BASE_URL.replace(/\\/+$/, ""); +const apiBase = process.env.AGATA_API_BASE_URL.replace(/\/+$/, ""); const apiKey = process.env.AGATA_API_KEY; const webhookSecret = process.env.AGATA_WEBHOOK_SIGNING_SECRET; const executorId = process.env.PROXIMA_DEPLOYMENT_EXECUTOR_ID; @@ -219,7 +219,7 @@ async function executeDeployment(event, deliveryId) { version, evidence: { ...baseEvidence, health_check: true, check: desiredState === "running" ? "engine-healthz" : "container-stopped" }, }); - await saveState({ + await saveState(stateFile, { ...state, last_known_good_version: version, completed: { @@ -242,7 +242,7 @@ async function executeDeployment(event, deliveryId) { evidence: { ...baseEvidence, rollback_version: previousVersion, rollback_health_check: true, health_check: true }, message: "The requested release failed its health check; the previous known-good release was restored.", }); - await saveState({ + await saveState(stateFile, { ...state, last_known_good_version: previousVersion, completed: { @@ -263,7 +263,7 @@ async function executeDeployment(event, deliveryId) { message: String(rolloutError.message || rolloutError).slice(0, 900), evidence: { ...baseEvidence, health_check: false, rollback_attempted: Boolean(previousVersion) }, }); - await saveState({ + await saveState(stateFile, { ...state, completed: { ...(state.completed || {}), From c4673842d513fd511b65ba1d9f2d54160ff18233 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:58:31 -0800 Subject: [PATCH 032/108] docs(engine): document same-host multi-node executor target map --- docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md index b751ded..de42807 100644 --- a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md +++ b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md @@ -26,6 +26,30 @@ The draining state is intentionally rejected because a safe drain operation requ The Engine image is published by the tagged-release workflow. For a tag such as v0.1.0, the workflow publishes both 0.1.0 and v0.1.0 plus latest. Make the ghcr.io/proxima-engine/proxima-engine package public before asking unauthenticated customer hosts to pull it; otherwise configure Docker registry authentication on the customer host. +One executor webhook is configured per organization. The reference executor can manage multiple registered Engine nodes when their Compose projects are on the same Docker host: configure a target map file and give each node its own project directory, Compose file, state directory and approved version list. For nodes on separate Docker hosts or clusters, use a customer-owned orchestrator/custom executor; this reference implementation does not manage remote Docker contexts. + +Example target map (save as targets.json): + +```json +{ + "00000000-0000-4000-8000-000000000001": { + "project_dir": "/opt/proxima/node-a", + "compose_file": "compose.customer.yaml", + "state_dir": "/var/lib/agata-deployment-executor/node-a", + "allowed_versions": ["0.1.0", "0.1.1"] + }, + "00000000-0000-4000-8000-000000000002": { + "project_dir": "/opt/proxima/node-b", + "compose_file": "compose.customer.yaml", + "state_dir": "/var/lib/agata-deployment-executor/node-b", + "allowed_versions": ["0.1.0"] + } +} +``` + +Set PROXIMA_DEPLOYMENT_TARGETS_FILE to this file in the executor environment. Each project directory must contain its own Compose file and .env configuration. If this variable is not set, the executor supports the simpler single-node PROXIMA_NODE_ID and PROXIMA_ENGINE_ALLOWLIST settings. + + ## Install Download this repository's ZIP and extract it on the same customer host that runs the Engine Compose project. Create a protected environment file for the executor. Do not commit it: @@ -37,6 +61,8 @@ AGATA_WEBHOOK_SIGNING_SECRET= PROXIMA_NODE_ID= PROXIMA_DEPLOYMENT_EXECUTOR_ID=customer-prod-engine-01 PROXIMA_ENGINE_ALLOWLIST=0.1.0,0.1.1 +# For a same-host multi-node fleet, replace the two single-node settings above with: +# PROXIMA_DEPLOYMENT_TARGETS_FILE=/etc/agata/targets.json PROXIMA_DEPLOYMENT_PROJECT_DIR=/opt/proxima PROXIMA_DEPLOYMENT_COMPOSE_FILE=compose.customer.yaml PROXIMA_DEPLOYMENT_STATE_DIR=/var/lib/agata-deployment-executor From 792afcb8b51165589325692e8627a044b73ef356 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:58:46 -0800 Subject: [PATCH 033/108] fix(engine): report rejected release and unsupported state as failed deployments --- deployment-executor/receiver.mjs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/deployment-executor/receiver.mjs b/deployment-executor/receiver.mjs index d99e0e0..0a95ca1 100644 --- a/deployment-executor/receiver.mjs +++ b/deployment-executor/receiver.mjs @@ -154,8 +154,6 @@ async function executeDeployment(event, deliveryId) { if (!/^[0-9a-f-]{36}$/i.test(deploymentId)) throw new Error("Deployment event has an invalid resource ID."); const target = targets.get(targetNode); if (!target) throw new Error("No customer deployment target is configured for this node ID."); - if (!target.allowedVersions.has(version)) throw new Error("Requested Engine version is not in this node target allowlist."); - if (!["running", "stopped"].includes(desiredState)) throw new Error("This reference executor supports running and stopped states; draining requires a configured drain controller."); const stateFile = target.stateFile; const state = await readState(stateFile); @@ -196,6 +194,29 @@ async function executeDeployment(event, deliveryId) { evidence: baseEvidence, }); + const rejection = !target.allowedVersions.has(version) + ? "Requested Engine version is not in this node target allowlist." + : !["running", "stopped"].includes(desiredState) + ? "This reference executor supports running and stopped states; draining requires a configured drain controller." + : null; + if (rejection) { + await reportResult(deploymentId, { + status: "failed", + observed_state: "unknown", + version, + message: rejection, + evidence: { ...baseEvidence, health_check: false }, + }); + await saveState(stateFile, { + ...state, + completed: { + ...(state.completed || {}), + [deploymentId]: { version, desired_state: desiredState, status: "failed", completed_at: new Date().toISOString() }, + }, + }); + return { accepted: true, failed: true, reason: rejection }; + } + const previousVersion = state.last_known_good_version && target.allowedVersions.has(state.last_known_good_version) ? state.last_known_good_version : null; From c325d5e7e39c1ee297f5dcc569732166a6f1a4e7 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:59:03 -0800 Subject: [PATCH 034/108] docs(engine): add multi-node executor target map template --- deployment-executor/targets.example.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 deployment-executor/targets.example.json diff --git a/deployment-executor/targets.example.json b/deployment-executor/targets.example.json new file mode 100644 index 0000000..063917e --- /dev/null +++ b/deployment-executor/targets.example.json @@ -0,0 +1,14 @@ +{ + "00000000-0000-4000-8000-000000000001": { + "project_dir": "/opt/proxima/node-a", + "compose_file": "compose.customer.yaml", + "state_dir": "/var/lib/agata-deployment-executor/node-a", + "allowed_versions": ["0.1.0", "0.1.1"] + }, + "00000000-0000-4000-8000-000000000002": { + "project_dir": "/opt/proxima/node-b", + "compose_file": "compose.customer.yaml", + "state_dir": "/var/lib/agata-deployment-executor/node-b", + "allowed_versions": ["0.1.0"] + } +} From be3b3f84fde8d1da6d36d24097cf9b2438adf29b Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 01:59:13 -0800 Subject: [PATCH 035/108] docs(engine): link multi-node target map template --- docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md index de42807..1ae7a21 100644 --- a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md +++ b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md @@ -28,7 +28,7 @@ The Engine image is published by the tagged-release workflow. For a tag such as One executor webhook is configured per organization. The reference executor can manage multiple registered Engine nodes when their Compose projects are on the same Docker host: configure a target map file and give each node its own project directory, Compose file, state directory and approved version list. For nodes on separate Docker hosts or clusters, use a customer-owned orchestrator/custom executor; this reference implementation does not manage remote Docker contexts. -Example target map (save as targets.json): +Copy [the multi-node target map template](../deployment-executor/targets.example.json) and replace its example UUIDs, paths and release allowlists with the customer's actual values. Each node must use a separate project directory so its Compose project name and persistent node identity volume remain isolated. Example target map (save as targets.json): ```json { From a37e41a9e720117c87189fc7f2d49dcf2a22ee6e Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:00:24 -0800 Subject: [PATCH 036/108] fix(engine): only mark running releases as rollback-safe versions --- deployment-executor/receiver.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment-executor/receiver.mjs b/deployment-executor/receiver.mjs index 0a95ca1..f4a1aca 100644 --- a/deployment-executor/receiver.mjs +++ b/deployment-executor/receiver.mjs @@ -174,7 +174,7 @@ async function executeDeployment(event, deliveryId) { } await saveState(stateFile, { ...state, - last_known_good_version: remote.applied_version || state.last_known_good_version, + last_known_good_version: remote.observed_state === "running" ? remote.applied_version || state.last_known_good_version : state.last_known_good_version, completed: { ...(state.completed || {}), [deploymentId]: { @@ -242,7 +242,7 @@ async function executeDeployment(event, deliveryId) { }); await saveState(stateFile, { ...state, - last_known_good_version: version, + last_known_good_version: desiredState === "running" ? version : state.last_known_good_version, completed: { ...(state.completed || {}), [deploymentId]: { version, desired_state: desiredState, status: "healthy", completed_at: new Date().toISOString() }, From 82fe51ca3863116762db2267890b5beef05a4678 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:03:44 -0800 Subject: [PATCH 037/108] feat(engine): provide systemd service template for customer executor --- .../agata-deployment-executor.service | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 deployment-executor/agata-deployment-executor.service diff --git a/deployment-executor/agata-deployment-executor.service b/deployment-executor/agata-deployment-executor.service new file mode 100644 index 0000000..69ed4b1 --- /dev/null +++ b/deployment-executor/agata-deployment-executor.service @@ -0,0 +1,25 @@ +[Unit] +Description=Agata Proxima customer-managed Engine deployment executor +After=network-online.target docker.service +Wants=network-online.target +Requires=docker.service + +[Service] +Type=simple +User=agata-deployer +Group=docker +WorkingDirectory=/opt/agata-deployment-executor +EnvironmentFile=/etc/agata/deployment-executor.env +ExecStart=/usr/bin/node /opt/agata-deployment-executor/deployment-executor/receiver.mjs +Restart=on-failure +RestartSec=5 +TimeoutStopSec=30 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/agata-deployment-executor +UMask=0077 + +[Install] +WantedBy=multi-user.target From aac04898b576ac1f2c870100175393bf97208b88 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:04:07 -0800 Subject: [PATCH 038/108] docs(engine): add Linux systemd executor installation procedure --- docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md index 1ae7a21..b2d643e 100644 --- a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md +++ b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md @@ -84,6 +84,24 @@ set +a node ./deployment-executor/receiver.mjs ``` +### Run as a Linux service + +The repository includes a systemd unit template at deployment-executor/agata-deployment-executor.service. After the smoke test succeeds, install the protected environment file and unit: + +```bash +sudo useradd --system --create-home --shell /usr/sbin/nologin agata-deployer +sudo usermod -aG docker agata-deployer +sudo install -d -o root -g root -m 0700 /etc/agata +sudo install -o root -g root -m 0600 deployment-executor.env /etc/agata/deployment-executor.env +sudo install -o root -g root -m 0644 deployment-executor/agata-deployment-executor.service /etc/systemd/system/agata-deployment-executor.service +sudo systemctl daemon-reload +sudo systemctl enable --now agata-deployment-executor +sudo systemctl status agata-deployment-executor +sudo journalctl -u agata-deployment-executor -f +``` + +The service user must be able to read each configured Compose project, .env file and secret file, and write to each configured executor state directory. The supplied unit allows writes to /var/lib/agata-deployment-executor; if your target map uses another state directory, add it to ReadWritePaths before starting the service. Membership in the docker group grants host-administrator-equivalent control; review that privilege and restrict the host accordingly. + The reverse proxy must preserve the raw request body and forward X-Agata-Signature, X-Agata-Delivery-Id, X-Agata-Event, and X-Agata-Event-Id. Keep its upstream request timeout at or above six minutes: Agata Proxima allows deployment.created delivery up to six minutes for image pulls, Engine restarts and runtime health checks. The executor returns only after the Control Plane has accepted the execution result. ## Configure Agata Proxima From 024f3887422cc360afadc5515ed28e44ff0aff07 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:04:44 -0800 Subject: [PATCH 039/108] docs(engine): document secret file ownership for non-root container --- docs/NODE_ENROLLMENT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/NODE_ENROLLMENT.md b/docs/NODE_ENROLLMENT.md index 6c0254b..a0c21f7 100644 --- a/docs/NODE_ENROLLMENT.md +++ b/docs/NODE_ENROLLMENT.md @@ -54,7 +54,7 @@ docker compose -f compose.customer.yaml up --build -d docker compose -f compose.customer.yaml logs -f engine ``` -The Compose file mounts the enrollment token and tenant signing key as read-only secret files. Neither secret is passed as a container environment variable. The Engine reads the enrollment token file only during first enrollment, persists its node identity in the named Docker volume, and sends signed heartbeats every 30 seconds. No heartbeat cron job, custom webhook receiver, or customer-written enrollment code is required. +The Compose file mounts the enrollment token and tenant signing key as read-only secret files. Neither secret is passed as a container environment variable. The Engine runs as UID 10001. On Linux, make the secret directory and files readable by that UID while keeping them private from other host users (for example, chown the directory and files to 10001:10001, chmod the directory 0700 and the files 0400). On Docker Desktop, confirm the mounted files are readable inside the Linux container; do not make the host secrets directory broadly readable just to work around a permission issue. The Engine reads the enrollment token file only during first enrollment, persists its node identity in the named Docker volume, and sends signed heartbeats every 30 seconds. No heartbeat cron job, custom webhook receiver, or customer-written enrollment code is required. When Fleet shows the node as healthy, replace the contents of secrets/node_enrollment_token with the harmless text enrollment-complete. The persisted node identity is used on subsequent starts, so the consumed one-time token is no longer needed. Keep secrets/tenant_signing_key protected and persistent. From efe1199151d1f2bd62ddf03394e7b04f66ea112c Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:09:09 -0800 Subject: [PATCH 040/108] docs(engine): document guided executor provisioning in Cloud --- docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md index b2d643e..a7ce47c 100644 --- a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md +++ b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md @@ -106,6 +106,9 @@ The reverse proxy must preserve the raw request body and forward X-Agata-Signatu ## Configure Agata Proxima +The recommended path is the guided setup in Deployments → Set up deployment executor. Select a healthy Engine node, enter the public HTTPS endpoint, executor identity, approved Engine versions and customer host paths. Agata Proxima creates the dedicated organization API key and signed deployment.created webhook, then returns a one-time deployment-executor.env file. Download it before leaving the result screen. If webhook creation fails after the API key is created, the wizard attempts to revoke the key. + +Manual setup is also available: 1. Open Developer → API keys and create a dedicated write-capable key for the executor. 2. Open Developer → Webhooks and create one HTTPS endpoint using the route above. 3. Subscribe to exactly deployment.created. Do not use * for this executor. From 5ce0a723afabb0a2ea34d80c8d0bc5aa398ad897 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:09:28 -0800 Subject: [PATCH 041/108] docs(engine): disclose executor API key privilege and rotation --- docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md index a7ce47c..0b9426e 100644 --- a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md +++ b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md @@ -106,7 +106,7 @@ The reverse proxy must preserve the raw request body and forward X-Agata-Signatu ## Configure Agata Proxima -The recommended path is the guided setup in Deployments → Set up deployment executor. Select a healthy Engine node, enter the public HTTPS endpoint, executor identity, approved Engine versions and customer host paths. Agata Proxima creates the dedicated organization API key and signed deployment.created webhook, then returns a one-time deployment-executor.env file. Download it before leaving the result screen. If webhook creation fails after the API key is created, the wizard attempts to revoke the key. +The recommended path is the guided setup in Deployments → Set up deployment executor. Select a healthy Engine node, enter the public HTTPS endpoint, executor identity, approved Engine versions and customer host paths. Agata Proxima creates a dedicated organization-scoped operator API key and signed deployment.created webhook, then returns a one-time deployment-executor.env file. The current API-key model grants operator-level organization access rather than deployment-only scopes; keep this key dedicated to the executor, store it as a protected secret, and revoke it when the executor is retired. Download it before leaving the result screen. If webhook creation fails after the API key is created, the wizard attempts to revoke the key. Manual setup is also available: 1. Open Developer → API keys and create a dedicated write-capable key for the executor. @@ -129,7 +129,7 @@ A failed running rollout attempts to restore the previous version only when that - This process can control Docker and therefore has high privilege on its host. Run it as a dedicated OS identity and restrict access to its environment file. - Access to the Docker socket or Docker group is effectively host-administrator access. Do not expose the Docker socket over TCP. -- Keep the API key, webhook secret, and node identity separate. Rotate the API key and webhook secret independently. +- Keep the API key, webhook secret, and node identity separate. The executor API key is an organization-scoped operator credential, so protect it accordingly. Rotate the API key and webhook secret independently. - Keep the executor state directory persistent and protected. It stores deployment IDs and release metadata, not API keys or webhook secrets. - The webhook receiver returns 503 when execution or callback delivery cannot be completed, allowing the Control Plane delivery queue to retry. - Test invalid signatures, duplicate delivery, image pull failure, health-check failure, rollback success and rollback failure in staging before production use. From 9dfb6d21acc8308b59b46087672bbfd801e755f2 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:10:10 -0800 Subject: [PATCH 042/108] fix(engine): bind deployment executor to loopback behind HTTPS proxy --- deployment-executor/receiver.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment-executor/receiver.mjs b/deployment-executor/receiver.mjs index f4a1aca..d6f330b 100644 --- a/deployment-executor/receiver.mjs +++ b/deployment-executor/receiver.mjs @@ -344,4 +344,4 @@ const server = createServer((req, res) => { server.requestTimeout = 360_000; server.headersTimeout = 365_000; -server.listen(port, () => console.log(`Agata Proxima deployment executor listening on port ${port}`)); +server.listen(port, "127.0.0.1", () => console.log(`Agata Proxima deployment executor listening on 127.0.0.1:${port}`)); From c2f164a65d49b9c40dd70d268db5b16643b972ec Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:10:15 -0800 Subject: [PATCH 043/108] docs(engine): add HTTPS reverse proxy example for executor --- deployment-executor/Caddyfile.example | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 deployment-executor/Caddyfile.example diff --git a/deployment-executor/Caddyfile.example b/deployment-executor/Caddyfile.example new file mode 100644 index 0000000..269d495 --- /dev/null +++ b/deployment-executor/Caddyfile.example @@ -0,0 +1,3 @@ +executor.example.com { + reverse_proxy 127.0.0.1:8789 +} From d7c291c6e017fbe2b1dfbc87f473b1338cbd8c47 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:10:21 -0800 Subject: [PATCH 044/108] docs(engine): document loopback listener and Caddy HTTPS proxy --- docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md index 0b9426e..85810e7 100644 --- a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md +++ b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md @@ -71,6 +71,16 @@ PORT=8789 The allowlist must contain only Engine releases the customer has reviewed and approved. Incoming webhook values are never added to it. +The executor binds to 127.0.0.1 only. Put a customer-managed HTTPS reverse proxy in front of it. For Caddy, copy [the example Caddyfile](../deployment-executor/Caddyfile.example), replace executor.example.com with a DNS name pointing to the customer host, and reload Caddy. Caddy obtains and renews the TLS certificate automatically when ports 80 and 443 are reachable. + +Example: + +```caddyfile +executor.example.com { + reverse_proxy 127.0.0.1:8789 +} +``` + Run the service behind a customer-managed HTTPS reverse proxy at: POST https:///webhooks/agata/deployments From e9a645dd3215b536429437aa58d3a26382de869f Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:12:05 -0800 Subject: [PATCH 045/108] fix(engine): run executor from the same Engine repository checkout --- deployment-executor/agata-deployment-executor.service | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment-executor/agata-deployment-executor.service b/deployment-executor/agata-deployment-executor.service index 69ed4b1..276737a 100644 --- a/deployment-executor/agata-deployment-executor.service +++ b/deployment-executor/agata-deployment-executor.service @@ -8,9 +8,9 @@ Requires=docker.service Type=simple User=agata-deployer Group=docker -WorkingDirectory=/opt/agata-deployment-executor +WorkingDirectory=/opt/proxima EnvironmentFile=/etc/agata/deployment-executor.env -ExecStart=/usr/bin/node /opt/agata-deployment-executor/deployment-executor/receiver.mjs +ExecStart=/usr/bin/node /opt/proxima/deployment-executor/receiver.mjs Restart=on-failure RestartSec=5 TimeoutStopSec=30 From c3db9721af6e0a82019310eeb4af817466907277 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:12:13 -0800 Subject: [PATCH 046/108] docs(engine): align executor install paths with one repository checkout --- docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md index 85810e7..ca5affa 100644 --- a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md +++ b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md @@ -52,7 +52,7 @@ Set PROXIMA_DEPLOYMENT_TARGETS_FILE to this file in the executor environment. Ea ## Install -Download this repository's ZIP and extract it on the same customer host that runs the Engine Compose project. Create a protected environment file for the executor. Do not commit it: +Download this repository's ZIP and extract it to /opt/proxima on the same customer host that runs the Engine Compose project. This is the default path used by the customer Node setup and supplied systemd unit; if you choose another directory, update PROXIMA_DEPLOYMENT_PROJECT_DIR and the systemd WorkingDirectory/ExecStart paths accordingly. Create a protected environment file for the executor. Do not commit it: ```dotenv AGATA_API_BASE_URL=https://api.agataproxima.com @@ -103,7 +103,7 @@ sudo useradd --system --create-home --shell /usr/sbin/nologin agata-deployer sudo usermod -aG docker agata-deployer sudo install -d -o root -g root -m 0700 /etc/agata sudo install -o root -g root -m 0600 deployment-executor.env /etc/agata/deployment-executor.env -sudo install -o root -g root -m 0644 deployment-executor/agata-deployment-executor.service /etc/systemd/system/agata-deployment-executor.service +sudo install -o root -g root -m 0644 /opt/proxima/deployment-executor/agata-deployment-executor.service /etc/systemd/system/agata-deployment-executor.service sudo systemctl daemon-reload sudo systemctl enable --now agata-deployment-executor sudo systemctl status agata-deployment-executor From c254c8c3647acd91734641ca6f2f675fc0b81b09 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:12:30 -0800 Subject: [PATCH 047/108] docs(engine): align service identity and secret permissions with Engine UID --- docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md index ca5affa..9726824 100644 --- a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md +++ b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md @@ -99,9 +99,15 @@ node ./deployment-executor/receiver.mjs The repository includes a systemd unit template at deployment-executor/agata-deployment-executor.service. After the smoke test succeeds, install the protected environment file and unit: ```bash -sudo useradd --system --create-home --shell /usr/sbin/nologin agata-deployer +getent passwd 10001 || sudo useradd --system --uid 10001 --create-home --shell /usr/sbin/nologin agata-deployer sudo usermod -aG docker agata-deployer sudo install -d -o root -g root -m 0700 /etc/agata +sudo install -d -o agata-deployer -g docker -m 0700 /var/lib/agata-deployment-executor +sudo chown agata-deployer:docker /opt/proxima/.env +sudo chmod 0640 /opt/proxima/.env +sudo chown -R agata-deployer:agata-deployer /opt/proxima/secrets +sudo chmod 0700 /opt/proxima/secrets +sudo chmod 0400 /opt/proxima/secrets/* sudo install -o root -g root -m 0600 deployment-executor.env /etc/agata/deployment-executor.env sudo install -o root -g root -m 0644 /opt/proxima/deployment-executor/agata-deployment-executor.service /etc/systemd/system/agata-deployment-executor.service sudo systemctl daemon-reload @@ -110,7 +116,7 @@ sudo systemctl status agata-deployment-executor sudo journalctl -u agata-deployment-executor -f ``` -The service user must be able to read each configured Compose project, .env file and secret file, and write to each configured executor state directory. The supplied unit allows writes to /var/lib/agata-deployment-executor; if your target map uses another state directory, add it to ReadWritePaths before starting the service. Membership in the docker group grants host-administrator-equivalent control; review that privilege and restrict the host accordingly. +The service user is assigned UID 10001 to match the non-root Engine container, so it can read the mounted secret files without making them world-readable. Confirm UID 10001 is available before creating the account. The service user must be able to read each configured Compose project and .env file, and write to each configured executor state directory. The supplied unit allows writes to /var/lib/agata-deployment-executor; if your target map uses another state directory, add it to ReadWritePaths and create it with agata-deployer ownership before starting the service. Membership in the docker group grants host-administrator-equivalent control; review that privilege and restrict the host accordingly. The reverse proxy must preserve the raw request body and forward X-Agata-Signature, X-Agata-Delivery-Id, X-Agata-Event, and X-Agata-Event-Id. Keep its upstream request timeout at or above six minutes: Agata Proxima allows deployment.created delivery up to six minutes for image pulls, Engine restarts and runtime health checks. The executor returns only after the Control Plane has accepted the execution result. From c1501ef8beacc63a306330fd7c777dcd271a5822 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:12:43 -0800 Subject: [PATCH 048/108] docs(engine): make executor service account setup fail safely on UID conflict --- docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md index 9726824..3e98724 100644 --- a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md +++ b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md @@ -99,7 +99,10 @@ node ./deployment-executor/receiver.mjs The repository includes a systemd unit template at deployment-executor/agata-deployment-executor.service. After the smoke test succeeds, install the protected environment file and unit: ```bash -getent passwd 10001 || sudo useradd --system --uid 10001 --create-home --shell /usr/sbin/nologin agata-deployer +if ! id agata-deployer >/dev/null 2>&1; then + if getent passwd 10001 >/dev/null; then echo 'UID 10001 is already in use; choose and configure a matching Engine UID before continuing.' >&2; exit 1; fi + sudo useradd --system --uid 10001 --create-home --shell /usr/sbin/nologin agata-deployer +fi sudo usermod -aG docker agata-deployer sudo install -d -o root -g root -m 0700 /etc/agata sudo install -d -o agata-deployer -g docker -m 0700 /var/lib/agata-deployment-executor From af941f10dadd064a1f40f1039f6d40d21d8fa7fb Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:15:38 -0800 Subject: [PATCH 049/108] fix(engine): satisfy Clippy for tenant secret length validation --- crates/proxima-engine/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/proxima-engine/src/config.rs b/crates/proxima-engine/src/config.rs index a34e0d4..7596863 100644 --- a/crates/proxima-engine/src/config.rs +++ b/crates/proxima-engine/src/config.rs @@ -84,7 +84,7 @@ impl Config { }; let tenant_signing_key = match tenant_signing_key_value { Some(value) if value.trim().is_empty() => None, - Some(value) if value.as_bytes().len() < 32 => { + Some(value) if value.len() < 32 => { return Err(io::Error::new( io::ErrorKind::InvalidInput, "PROXIMA_TENANT_SIGNING_KEY or its secret file must contain at least 32 bytes", From 0306b1015113892e33279aa628b95e947d3cf37b Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:21:57 -0800 Subject: [PATCH 050/108] feat(onboarding): add guided Windows customer Engine installer --- install-customer.ps1 | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 install-customer.ps1 diff --git a/install-customer.ps1 b/install-customer.ps1 new file mode 100644 index 0000000..12b8179 --- /dev/null +++ b/install-customer.ps1 @@ -0,0 +1,40 @@ +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest + +$root = Split-Path -Parent $MyInvocation.MyCommand.Path +Set-Location $root + +$required = @(".env", "compose.customer.yaml", "secrets/node_enrollment_token", "secrets/tenant_signing_key") +foreach ($path in $required) { + if (-not (Test-Path -LiteralPath (Join-Path $root $path) -PathType Leaf)) { + throw "Missing required customer setup file: $path. Download .env and both secret files from Agata Proxima → Nodes, then place them in this folder." + } +} +if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { + throw "Docker CLI was not found. Install Docker Desktop or Docker Engine with the Compose v2 plugin, then rerun this script." +} +$version = docker compose version +if ($LASTEXITCODE -ne 0) { throw "Docker Compose v2 is required." } +Write-Host $version + +docker compose -f compose.customer.yaml config --quiet +if ($LASTEXITCODE -ne 0) { throw "Customer Compose configuration is invalid. Check .env and secret file paths." } + +Write-Host "Building and starting Proxima Engine. The first build downloads Rust dependencies and can take several minutes." +docker compose -f compose.customer.yaml up -d --build +if ($LASTEXITCODE -ne 0) { throw "Engine container did not start. Inspect with: docker compose -f compose.customer.yaml logs --tail 100 engine" } + +Write-Host "Waiting for the Engine health check..." +$healthy = $false +for ($attempt = 0; $attempt -lt 40; $attempt++) { + $status = docker compose -f compose.customer.yaml ps --format json engine 2>$null + if ($LASTEXITCODE -eq 0 -and $status -match '"Health":"healthy"') { $healthy = $true; break } + Start-Sleep -Seconds 3 +} +if (-not $healthy) { + docker compose -f compose.customer.yaml ps + docker compose -f compose.customer.yaml logs --tail 100 engine + throw "Engine did not become healthy within 120 seconds. Review the logs above before retrying." +} +Write-Host "Proxima Engine is healthy. Node enrollment and signed heartbeat will now complete against the configured Agata Proxima Control Plane." +docker compose -f compose.customer.yaml ps From 09cdb4f1e40f1f4dd7a89e18b0681adecb10ac16 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:22:00 -0800 Subject: [PATCH 051/108] feat(onboarding): add guided Linux customer Engine installer --- install-customer.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 install-customer.sh diff --git a/install-customer.sh b/install-customer.sh new file mode 100644 index 0000000..ce2e3b5 --- /dev/null +++ b/install-customer.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +cd "$(dirname "$0")" + +for file in .env compose.customer.yaml secrets/node_enrollment_token secrets/tenant_signing_key; do + if [[ ! -f "$file" ]]; then + echo "Missing $file. Download .env and both secret files from Agata Proxima → Nodes, then place them in this folder." >&2 + exit 1 + fi +done +command -v docker >/dev/null 2>&1 || { echo "Docker CLI is required." >&2; exit 1; } +docker compose version >/dev/null + +docker compose -f compose.customer.yaml config --quiet +echo "Building and starting Proxima Engine. The first build downloads Rust dependencies and can take several minutes." +docker compose -f compose.customer.yaml up -d --build + +echo "Waiting for Engine health check..." +for attempt in $(seq 1 40); do + if docker compose -f compose.customer.yaml ps --format json engine | grep -q '"Health":"healthy"'; then + docker compose -f compose.customer.yaml ps + echo "Proxima Engine is healthy. Node enrollment and signed heartbeat will now complete against the configured Agata Proxima Control Plane." + exit 0 + fi + sleep 3 +done +docker compose -f compose.customer.yaml ps +docker compose -f compose.customer.yaml logs --tail 100 engine +echo "Engine did not become healthy within 120 seconds. Review the logs before retrying." >&2 +exit 1 From d5892dde680bea23ece4aa0f9ff739c265090aa3 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:22:10 -0800 Subject: [PATCH 052/108] docs(onboarding): add customer node install steps for Windows and Linux --- docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md index 3e98724..f4e411c 100644 --- a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md +++ b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md @@ -50,6 +50,31 @@ Copy [the multi-node target map template](../deployment-executor/targets.example Set PROXIMA_DEPLOYMENT_TARGETS_FILE to this file in the executor environment. Each project directory must contain its own Compose file and .env configuration. If this variable is not set, the executor supports the simpler single-node PROXIMA_NODE_ID and PROXIMA_ENGINE_ALLOWLIST settings. +## Install the Engine (customer node) + +The dashboard gives the customer a node-specific `.env`, a one-time enrollment-token file and a tenant-signing-key file. The customer does not need to hand-write Docker Compose configuration or paste secrets into shell commands. + +1. Install Docker Desktop (Windows) or Docker Engine with the Compose v2 plugin (Linux). +2. Download the Engine ZIP from the node setup screen and extract it. +3. In the extracted folder, create a `secrets` directory. +4. Place the downloaded `.env` in the repository root. Place the one-time token at `secrets/node_enrollment_token` and the tenant signing key at `secrets/tenant_signing_key`. +5. Run the supplied installer from that folder: + + **Windows PowerShell** + ```powershell + .\\install-customer.ps1 + ``` + + **Linux** + ```bash + chmod +x ./install-customer.sh + ./install-customer.sh + ``` + +The installer validates the required files and Docker Compose configuration, builds the Engine image locally, starts the container, and waits for its real Docker health check. It prints the container logs if startup fails. The Engine consumes the one-time enrollment token from the mounted secret, persists its node identity in a Docker volume, and begins signed heartbeats. The token is not needed again after successful enrollment. Keep the `.env` and `secrets` directory private; never commit or email them. + +The Engine listens on port 6432 and binds to loopback by default. Keep that binding when the application runs on the same host. If the application is on another host or container network, deliberately configure a private reachable bind address/network and firewall policy; do not expose PostgreSQL proxy traffic to the public internet. + ## Install Download this repository's ZIP and extract it to /opt/proxima on the same customer host that runs the Engine Compose project. This is the default path used by the customer Node setup and supplied systemd unit; if you choose another directory, update PROXIMA_DEPLOYMENT_PROJECT_DIR and the systemd WorkingDirectory/ExecStart paths accordingly. Create a protected environment file for the executor. Do not commit it: From c9c9ff93b254c116e770e3f890a22b86e79caf34 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:22:35 -0800 Subject: [PATCH 053/108] ci(onboarding): validate customer installer scripts --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9988bc6..6f254ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,10 @@ jobs: run: python3 -m unittest discover -s tests_cli -v - name: Validate customer deployment executor syntax run: node --check deployment-executor/receiver.mjs + - name: Validate customer installer scripts + run: | + bash -n install-customer.sh + pwsh -NoProfile -Command '$tokens = $null; $errors = $null; [System.Management.Automation.Language.Parser]::ParseFile((Resolve-Path "install-customer.ps1"), [ref]$tokens, [ref]$errors) > $null; if ($errors.Count -gt 0) { $errors | ForEach-Object { Write-Error $_ }; exit 1 }' - name: Validate customer Compose installation contract run: | mkdir -p secrets From 2e829bede5b38588e76e304358122ccae98e0af0 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:26:59 -0800 Subject: [PATCH 054/108] feat(deployments): add guided Linux executor service installer --- install-deployment-executor.sh | 62 ++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 install-deployment-executor.sh diff --git a/install-deployment-executor.sh b/install-deployment-executor.sh new file mode 100644 index 0000000..616828a --- /dev/null +++ b/install-deployment-executor.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_SOURCE="$ROOT/deployment-executor.env" +UNIT_SOURCE="$ROOT/deployment-executor/agata-deployment-executor.service" +RECEIVER="$ROOT/deployment-executor/receiver.mjs" + +if [[ "$(id -u)" -ne 0 ]]; then + echo "Run this installer with sudo: sudo ./install-deployment-executor.sh" >&2 + exit 1 +fi +if [[ "$ROOT" != "/opt/proxima" ]]; then + echo "Place the reviewed Engine bundle in /opt/proxima before installing the executor. Current directory: $ROOT" >&2 + exit 1 +fi +for file in "$ENV_SOURCE" "$UNIT_SOURCE" "$RECEIVER"; do + [[ -f "$file" ]] || { echo "Missing required file: $file" >&2; exit 1; } +done +command -v node >/dev/null 2>&1 || { echo "Install Node.js 22 or newer first." >&2; exit 1; } +node -e 'if (Number(process.versions.node.split(".")[0]) < 22) process.exit(1)' || { echo "Node.js 22 or newer is required." >&2; exit 1; } +command -v docker >/dev/null 2>&1 || { echo "Docker Engine is required." >&2; exit 1; } +docker compose version >/dev/null +command -v systemctl >/dev/null 2>&1 || { echo "This installer requires systemd." >&2; exit 1; } +for key in AGATA_API_BASE_URL AGATA_API_KEY AGATA_WEBHOOK_SIGNING_SECRET PROXIMA_DEPLOYMENT_EXECUTOR_ID; do + grep -Eq "^${key}=.+" "$ENV_SOURCE" || { echo "deployment-executor.env is missing $key. Download it from Deployments → Set up deployment executor." >&2; exit 1; } +done +grep -Eq '^PROXIMA_NODE_ID=.+' "$ENV_SOURCE" && grep -Eq '^PROXIMA_ENGINE_ALLOWLIST=.+' "$ENV_SOURCE" || grep -Eq '^PROXIMA_DEPLOYMENT_TARGETS_FILE=.+' "$ENV_SOURCE" || { + echo "Configure PROXIMA_NODE_ID + PROXIMA_ENGINE_ALLOWLIST or PROXIMA_DEPLOYMENT_TARGETS_FILE in deployment-executor.env." >&2 + exit 1 +} + +if ! id agata-deployer >/dev/null 2>&1; then + if getent passwd 10001 >/dev/null; then + echo "UID 10001 is already in use. Resolve the UID conflict before installing." >&2 + exit 1 + fi + useradd --system --uid 10001 --create-home --shell /usr/sbin/nologin agata-deployer +fi +getent group docker >/dev/null || { echo "Docker group not found. Install Docker Engine first." >&2; exit 1; } +usermod -aG docker agata-deployer +install -d -o root -g root -m 0700 /etc/agata +install -d -o agata-deployer -g docker -m 0700 /var/lib/agata-deployment-executor +install -o root -g root -m 0600 "$ENV_SOURCE" /etc/agata/deployment-executor.env +install -o root -g root -m 0644 "$UNIT_SOURCE" /etc/systemd/system/agata-deployment-executor.service + +if [[ -f "$ROOT/.env" ]]; then + chown agata-deployer:docker "$ROOT/.env" + chmod 0640 "$ROOT/.env" +fi +if [[ -d "$ROOT/secrets" ]]; then + chown -R agata-deployer:agata-deployer "$ROOT/secrets" + chmod 0700 "$ROOT/secrets" + find "$ROOT/secrets" -type f -exec chmod 0400 {} + +fi + +systemctl daemon-reload +systemctl enable --now agata-deployment-executor +systemctl --no-pager --full status agata-deployment-executor +echo +echo "Executor service installed. Configure a public HTTPS reverse proxy for POST /webhooks/agata/deployments before enabling deployments in Agata Proxima." +echo "Review the Docker-group privilege and keep /etc/agata/deployment-executor.env restricted to root." From f28a185274d4cdc5041fdc43a4ea26856be9f076 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:27:08 -0800 Subject: [PATCH 055/108] docs(deployments): document guided Linux executor service installer --- docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md index f4e411c..21ad59a 100644 --- a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md +++ b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md @@ -121,7 +121,15 @@ node ./deployment-executor/receiver.mjs ### Run as a Linux service -The repository includes a systemd unit template at deployment-executor/agata-deployment-executor.service. After the smoke test succeeds, install the protected environment file and unit: +For the standard single-host installation, place the downloaded `deployment-executor.env` in `/opt/proxima` and run the supplied installer: + +```bash +sudo ./install-deployment-executor.sh +``` + +The installer checks Node.js 22+, Docker Compose, the executor environment file and the service unit; creates the restricted service account; installs the environment file and systemd unit with protected permissions; grants the service access to the Docker socket; and enables the service. It does not configure the public HTTPS reverse proxy or DNS. Configure those separately before creating deployments. + +For manual installation or a non-standard host, the repository includes a systemd unit template at `deployment-executor/agata-deployment-executor.service`. The equivalent manual steps are: ```bash if ! id agata-deployer >/dev/null 2>&1; then From 3d328bab28f603a4100d2f28ab4b67d16d4a3a42 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:27:16 -0800 Subject: [PATCH 056/108] ci(deployments): validate executor service installer syntax --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f254ef..cc1cc62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,7 @@ jobs: - name: Validate customer installer scripts run: | bash -n install-customer.sh + bash -n install-deployment-executor.sh pwsh -NoProfile -Command '$tokens = $null; $errors = $null; [System.Management.Automation.Language.Parser]::ParseFile((Resolve-Path "install-customer.ps1"), [ref]$tokens, [ref]$errors) > $null; if ($errors.Count -gt 0) { $errors | ForEach-Object { Write-Error $_ }; exit 1 }' - name: Validate customer Compose installation contract run: | From 9d6e723f7a34cabbc1cbe4c564473d722d2b2035 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:29:49 -0800 Subject: [PATCH 057/108] docs(readme): surface guided Windows and Linux customer install --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4078bf0..4a29b9d 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ LinkedIn

-**Start:** [What is Agata Proxima?](#what-is-agata-proxima) · [Customer Docker install](docs/NODE_ENROLLMENT.md#2-one-command-docker-installation-hosted-cloud) · [Deployment executor](docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md) · [Launch evaluation](docs/LAUNCH.md) · [Quickstart](docs/QUICKSTART.md) · [Local developer mode](docs/LOCAL_DEVELOPMENT.md) · [Engine CLI](docs/CLI.md) · [Documentation index](docs/README.md) +**Start:** [What is Agata Proxima?](#what-is-agata-proxima) · [Customer Docker install](docs/NODE_ENROLLMENT.md#2-one-command-docker-installation-hosted-cloud) · [Guided customer install (Windows/Linux)](docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md#install-the-engine-customer-node) · [Deployment executor](docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md) · [Launch evaluation](docs/LAUNCH.md) · [Quickstart](docs/QUICKSTART.md) · [Local developer mode](docs/LOCAL_DEVELOPMENT.md) · [Engine CLI](docs/CLI.md) · [Documentation index](docs/README.md) **Technical guides:** [Architecture](docs/ARCHITECTURE.md) · [Tenant isolation](docs/TENANT_ISOLATION.md) · [Driver integrations](docs/INTEGRATIONS.md) · [API and metrics](docs/API.md) · [Security](docs/SECURITY.md) · [Deployment](docs/DEPLOYMENT.md) · [Verification](docs/VERIFICATION.md) From 832fe9eccfc5ac3bf086b3fd1b203998cc7fe89c Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:34:05 -0800 Subject: [PATCH 058/108] fix(deployments): allow longer customer image pulls during rollout --- deployment-executor/receiver.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment-executor/receiver.mjs b/deployment-executor/receiver.mjs index d6f330b..2e1244a 100644 --- a/deployment-executor/receiver.mjs +++ b/deployment-executor/receiver.mjs @@ -89,7 +89,7 @@ function docker(target, args, version) { cwd: target.projectDir, env: { ...process.env, PROXIMA_ENGINE_VERSION: version }, encoding: "utf8", - timeout: 120_000, + timeout: 240_000, maxBuffer: 1024 * 1024, shell: false, }); From 4a1ac1ba86403e189d3a163751a6c37fad6f176c Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:37:48 -0800 Subject: [PATCH 059/108] fix(deployments): apply selected release image while keeping Engine stopped --- deployment-executor/receiver.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deployment-executor/receiver.mjs b/deployment-executor/receiver.mjs index 2e1244a..afc53bf 100644 --- a/deployment-executor/receiver.mjs +++ b/deployment-executor/receiver.mjs @@ -229,6 +229,9 @@ async function executeDeployment(event, deliveryId) { if (!healthy) throw new Error("Engine did not pass its local /healthz check after rollout."); } else { docker(target, ["stop", "engine"], version); + // Recreate the stopped container from the newly pulled image so the reported + // applied version matches the container that will start on the next boot. + docker(target, ["create", "--no-deps", "--force-recreate", "engine"], version); const runningServices = docker(target, ["ps", "--status", "running", "--services"], version) .split(/\r?\n/).filter(Boolean); if (runningServices.includes("engine")) throw new Error("Engine container is still running after the stop request."); From 29c1c04357f554d0f87a1572228491b40d968201 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 02:38:06 -0800 Subject: [PATCH 060/108] docs(engine): clarify stopped release application semantics --- docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md index 21ad59a..5e2b73b 100644 --- a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md +++ b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md @@ -8,8 +8,8 @@ This reference executor connects Agata Proxima's signed deployment event to a cu 2. Verifies the exact raw-body HMAC-SHA256 signature and five-minute timestamp window. 3. Rejects events for a different node or a release version outside the customer's explicit allowlist. 4. Sends the applying callback with a stable executor and execution ID. -5. Pulls the requested versioned Engine image and applies the requested running or stopped state using fixed Docker Compose arguments. It never executes command text from a webhook. -6. Checks the real container state. A running Engine must pass its local /healthz endpoint. +5. Pulls the requested versioned Engine image and applies the requested state using fixed Docker Compose arguments. For `running`, it starts the Engine; for `stopped`, it stops and recreates the container without starting it, so the applied version matches the image configured for the next start. It never executes command text from a webhook. +6. Checks the real container state. A running Engine must pass its local /healthz endpoint, and a stopped Engine must not appear among running services. 7. Sends healthy only after the check passes. If a running rollout fails and a prior known-good release is recorded, it restores that release and reports rolled_back. 8. Persists completed deployment IDs locally. It can recover from a lost local completion marker by inspecting the authoritative Control Plane deployment result before retrying side effects. From 0e58b227446541ea204e7e77b7d6c9380e47a226 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 03:54:08 -0800 Subject: [PATCH 061/108] Harden Windows customer installer health checks --- install-customer.ps1 | 50 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/install-customer.ps1 b/install-customer.ps1 index 12b8179..1805df0 100644 --- a/install-customer.ps1 +++ b/install-customer.ps1 @@ -10,31 +10,61 @@ foreach ($path in $required) { throw "Missing required customer setup file: $path. Download .env and both secret files from Agata Proxima → Nodes, then place them in this folder." } } + if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { - throw "Docker CLI was not found. Install Docker Desktop or Docker Engine with the Compose v2 plugin, then rerun this script." + throw "Docker CLI was not found. Install Docker Desktop with the WSL 2 backend and Docker Compose v2, then rerun this script." +} + +$dockerOs = docker info --format '{{.OSType}}' +if ($LASTEXITCODE -ne 0) { + throw "Docker Desktop is not running or the Docker daemon is unavailable. Start Docker Desktop, wait until it reports Engine running, and rerun this script." } +if ($dockerOs.Trim() -ne "linux") { + throw "Proxima Engine uses a Linux container image. In Docker Desktop, switch to Linux containers and rerun this installer." +} + $version = docker compose version -if ($LASTEXITCODE -ne 0) { throw "Docker Compose v2 is required." } +if ($LASTEXITCODE -ne 0) { throw "Docker Compose v2 is required. Update Docker Desktop and confirm that 'docker compose version' works." } Write-Host $version -docker compose -f compose.customer.yaml config --quiet -if ($LASTEXITCODE -ne 0) { throw "Customer Compose configuration is invalid. Check .env and secret file paths." } +$configOutput = docker compose -f compose.customer.yaml config --quiet 2>&1 +if ($LASTEXITCODE -ne 0) { + throw "Customer Compose configuration is invalid. Check .env and the two secret-file paths. Details: $configOutput" +} Write-Host "Building and starting Proxima Engine. The first build downloads Rust dependencies and can take several minutes." docker compose -f compose.customer.yaml up -d --build -if ($LASTEXITCODE -ne 0) { throw "Engine container did not start. Inspect with: docker compose -f compose.customer.yaml logs --tail 100 engine" } +if ($LASTEXITCODE -ne 0) { + docker compose -f compose.customer.yaml logs --tail 100 engine + throw "Engine container did not start. Review the logs above, correct the reported issue, and rerun this installer." +} -Write-Host "Waiting for the Engine health check..." +Write-Host "Waiting up to 120 seconds for Docker's Engine health check..." $healthy = $false for ($attempt = 0; $attempt -lt 40; $attempt++) { - $status = docker compose -f compose.customer.yaml ps --format json engine 2>$null - if ($LASTEXITCODE -eq 0 -and $status -match '"Health":"healthy"') { $healthy = $true; break } + $containerId = docker compose -f compose.customer.yaml ps -q engine + if ($LASTEXITCODE -eq 0 -and -not [string]::IsNullOrWhiteSpace(($containerId | Out-String).Trim())) { + $health = docker inspect --format '{{.State.Health.Status}}' $containerId + if ($LASTEXITCODE -eq 0 -and $health.Trim() -eq "healthy") { + $healthy = $true + break + } + if ($LASTEXITCODE -eq 0 -and $health.Trim() -eq "unhealthy") { + break + } + } Start-Sleep -Seconds 3 } + if (-not $healthy) { docker compose -f compose.customer.yaml ps docker compose -f compose.customer.yaml logs --tail 100 engine - throw "Engine did not become healthy within 120 seconds. Review the logs above before retrying." + throw "Engine did not pass its Docker health check within 120 seconds. Review the container status and logs above before retrying." } -Write-Host "Proxima Engine is healthy. Node enrollment and signed heartbeat will now complete against the configured Agata Proxima Control Plane." + +Write-Host "" +Write-Host "Proxima Engine is healthy." +Write-Host "PostgreSQL proxy: 127.0.0.1:6432" +Write-Host "Node enrollment and signed heartbeat now run from the Engine container." +Write-Host "Confirm that the registered node appears as healthy in Agata Proxima Fleet." docker compose -f compose.customer.yaml ps From 6330180cb7aef80f43e90f97f7cca73af0422cad Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 03:54:17 -0800 Subject: [PATCH 062/108] Add Windows installer smoke test --- tests/windows-installer-smoke.ps1 | 51 +++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/windows-installer-smoke.ps1 diff --git a/tests/windows-installer-smoke.ps1 b/tests/windows-installer-smoke.ps1 new file mode 100644 index 0000000..bb8d163 --- /dev/null +++ b/tests/windows-installer-smoke.ps1 @@ -0,0 +1,51 @@ +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest + +$sourceRoot = Split-Path -Parent (Split-Path -Parent $MyInvocation.MyCommand.Path) +$tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("proxima-windows-installer-test-" + [guid]::NewGuid().ToString("N")) +New-Item -ItemType Directory -Path (Join-Path $tempRoot "secrets") -Force | Out-Null +Copy-Item (Join-Path $sourceRoot "install-customer.ps1") $tempRoot +Copy-Item (Join-Path $sourceRoot "compose.customer.yaml") $tempRoot +Set-Content -LiteralPath (Join-Path $tempRoot ".env") -Value ("PROXIMA_NODE_ID=00000000-0000-4000-8000-000000000001" + [Environment]::NewLine + "PROXIMA_UPSTREAM_ADDR=postgres:5432") -NoNewline +Set-Content -LiteralPath (Join-Path $tempRoot "secrets/node_enrollment_token") -Value "test-one-time-token" -NoNewline +Set-Content -LiteralPath (Join-Path $tempRoot "secrets/tenant_signing_key") -Value "0123456789abcdef0123456789abcdef" -NoNewline + +$script:dockerCalls = [System.Collections.Generic.List[string]]::new() +function global:docker { + $call = ($args | ForEach-Object { [string]$_ }) -join " " + $script:dockerCalls.Add($call) + $global:LASTEXITCODE = 0 + + if ($call -eq "info --format {{.OSType}}") { return "linux" } + if ($call -eq "compose version") { return "Docker Compose version v2.30.0" } + if ($call -eq "compose -f compose.customer.yaml config --quiet") { return } + if ($call -eq "compose -f compose.customer.yaml up -d --build") { return } + if ($call -eq "compose -f compose.customer.yaml ps -q engine") { return "test-engine-container" } + if ($call -eq "inspect --format {{.State.Health.Status}} test-engine-container") { return "healthy" } + if ($call -eq "compose -f compose.customer.yaml ps") { return "engine running (healthy)" } + throw "Unexpected Docker invocation in Windows installer test: $call" +} + +try { + & (Join-Path $tempRoot "install-customer.ps1") + + $requiredCalls = @( + "info --format {{.OSType}}", + "compose version", + "compose -f compose.customer.yaml config --quiet", + "compose -f compose.customer.yaml up -d --build", + "compose -f compose.customer.yaml ps -q engine", + "inspect --format {{.State.Health.Status}} test-engine-container" + ) + foreach ($requiredCall in $requiredCalls) { + if (-not $script:dockerCalls.Contains($requiredCall)) { + throw "Windows installer did not perform required Docker operation: $requiredCall" + } + } + + Write-Host "PASS: Windows PowerShell installer validates prerequisites, Linux-container mode, Compose configuration, startup and container health." +} +finally { + Remove-Item Function:\docker -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue +} From dbcdafa545c7b39de011a83a3a888cd737e6ca0c Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 03:54:27 -0800 Subject: [PATCH 063/108] Add Windows installer CI validation --- .github/workflows/ci.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc1cc62..0158a05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,3 +65,24 @@ jobs: run: docker build -t proxima-engine:ci . - name: Verify PgBouncer tenant-pool isolation run: bash scripts/test-pgbouncer-pooling.sh + + + windows-installer: + name: Windows customer installer + runs-on: windows-latest + steps: + - name: Check out source + uses: actions/checkout@v4 + - name: Parse Windows installer + shell: pwsh + run: | + $tokens = $null + $errors = $null + [System.Management.Automation.Language.Parser]::ParseFile((Resolve-Path "install-customer.ps1"), [ref]$tokens, [ref]$errors) > $null + if ($errors.Count -gt 0) { + $errors | ForEach-Object { Write-Error $_ } + exit 1 + } + - name: Execute Windows installer smoke test with Docker host simulation + shell: pwsh + run: ./tests/windows-installer-smoke.ps1 From 47bcb09939ffb2fad268b919ed97ce90d25b8e8b Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 03:55:15 -0800 Subject: [PATCH 064/108] Document Windows and Linux customer installation in README --- README.md | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/README.md b/README.md index 4a29b9d..7b43588 100644 --- a/README.md +++ b/README.md @@ -162,3 +162,91 @@ The Proxima Engine source and covered documentation are distributed under the [M Use [GitHub Issues](https://github.com/Agata-Proxima/proxima-engine/issues) for reproducible Engine bugs and feature requests. Read [Contributing](CONTRIBUTING.md) before submitting a pull request. For vulnerabilities, use the private reporting process in [SECURITY.md](SECURITY.md).

Built by Agata Proxima · agataproxima.com · GitHub organization

+ +## Install a registered Engine node on Windows or Linux + +The customer-node installer supports Windows 10/11 and Windows Server hosts that run Docker Desktop or Docker Engine with **Linux containers** and Docker Compose v2. Windows users have a dedicated PowerShell installer; Linux users have a Bash installer. Both install the same Proxima Engine container and use the same signed enrollment and heartbeat protocol. + +### Windows downloads + +1. Download the [Proxima Engine source ZIP](https://github.com/Proxima-Engine/proxima-engine/archive/refs/heads/feat/cloud-node-heartbeat-20261110.zip) and extract it. +2. Download the node-specific `.env`, `node_enrollment_token`, and `tenant_signing_key` files from Agata Proxima Cloud → Node Management → the registered node's setup/download section. These are generated credentials; they are not public repository files. +3. Download the [Windows PowerShell installer](https://raw.githubusercontent.com/Proxima-Engine/proxima-engine/feat/cloud-node-heartbeat-20261110/install-customer.ps1) if it is not already present in the ZIP. The [customer Compose file](https://raw.githubusercontent.com/Proxima-Engine/proxima-engine/feat/cloud-node-heartbeat-20261110/compose.customer.yaml) must remain beside it. +4. In the extracted folder, create a directory named `secrets`. Place the downloaded secret files there using the exact extensionless filenames: + - `secrets/node_enrollment_token` + - `secrets/tenant_signing_key` +5. Place the downloaded `.env` in the same folder as `install-customer.ps1` and `compose.customer.yaml`. Keep this folder private. In File Explorer, enable **File name extensions** so Windows does not silently append `.txt` to the secret filenames. + +### Windows prerequisites + +- Windows 10/11 or Windows Server with a supported Docker installation. +- Docker Desktop configured to use the WSL 2 backend and Linux containers, or Docker Engine with Compose v2 on the Windows host. +- Docker daemon running before installation. +- The node-specific `.env` and both secret files from Node Management. +- Network access from the Engine container to the configured PostgreSQL upstream and Agata Proxima HTTPS enrollment/heartbeat endpoints. + +The Engine is distributed as a Linux container. On Windows, Docker Desktop runs that container through its Linux/WSL 2 backend; this is the supported Windows installation path. The PowerShell installer is a Windows host installer, not an MSI and not a native Windows service binary. + +### Run the Windows installer + +Open PowerShell in the extracted folder. If the machine's execution policy blocks a locally downloaded script, allow it for this PowerShell process only: + +```powershell +Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass +.\install-customer.ps1 +``` + +The installer performs these checks and actions in order: + +1. Confirms that `.env`, `compose.customer.yaml`, and both secret files exist. +2. Confirms the Docker CLI and a running Docker daemon. +3. Confirms that Docker is using Linux containers and Docker Compose v2 is available. +4. Validates the customer Compose configuration. +5. Builds and starts the Engine container. +6. Waits up to 120 seconds for the container's Docker health check to report `healthy`. +7. Prints container status and the local PostgreSQL proxy endpoint. + +A successful installation prints **Proxima Engine is healthy**. Then open Agata Proxima → Node Management/Fleet and confirm the registered node reports healthy after its signed heartbeat arrives. The one-time enrollment token is consumed during first enrollment; the persistent node identity is stored in a named Docker volume. Keep the tenant signing key mounted and protected for subsequent starts. + +### Verify and operate on Windows + +```powershell +docker compose -f compose.customer.yaml ps +docker compose -f compose.customer.yaml logs --tail 100 engine +Invoke-RestMethod http://127.0.0.1:6432 +``` + +The last command is not a PostgreSQL query test; use a PostgreSQL driver or `psql` configured for `127.0.0.1:6432` to verify SQL connectivity. For the complete disposable local SQL walkthrough, follow [Quickstart](QUICKSTART.md). + +Restart and stop the registered node: + +```powershell +docker compose -f compose.customer.yaml restart engine +docker compose -f compose.customer.yaml down +``` + +Start it again without rebuilding: + +```powershell +docker compose -f compose.customer.yaml up -d +``` + +### Linux installation + +Linux customers use the matching Bash installer after placing the same four configuration/secret files in the extracted repository folder: + +```bash +chmod +x ./install-customer.sh +./install-customer.sh +``` + +For the Linux deployment-executor service installer and its security requirements, see [Customer-managed deployment executor](CUSTOMER_DEPLOYMENT_EXECUTOR.md). + +### Network and security requirements + +- The Engine listens on TCP 6432 and binds to host loopback by default. Keep this binding when the application runs on the same host. +- If the application runs in another container, attach it to the configured Compose network and use the Engine service name and port. For a different host, use a private network/VPN, a deliberate private bind address, and a restrictive firewall rule. +- The configured PostgreSQL upstream address is resolved from inside the Engine container. `localhost` refers to the container itself, not the Windows host. Use a reachable database hostname or the appropriate Docker Desktop host gateway address. +- Never expose PostgreSQL proxy port 6432, the status interface, or metrics publicly. +- Never commit, email, or paste `.env`, enrollment tokens, tenant signing keys, node identity files, operator API keys, or webhook secrets. +- The tenant signing key must match the key used by the trusted application backend to sign tenant-context tokens. It is distinct from the one-time enrollment token and the node identity signing key. From 17ca8059f1c5f33d021d1718d483ff885b6369d6 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 03:55:25 -0800 Subject: [PATCH 065/108] Add Windows and Linux registered-node quickstart --- docs/QUICKSTART.md | 70 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 9a5f46b..963f6f7 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -142,3 +142,73 @@ Connect your PostgreSQL client to port 6432. The Engine reads configuration from - [ ] Complete the [tenant isolation guide](TENANT_ISOLATION.md). - [ ] Connect an existing application using [driver integration recipes](INTEGRATIONS.md). - [ ] Review [deployment](DEPLOYMENT.md), [security](SECURITY.md), and [operations](OPERATIONS.md) before routing sensitive workloads. + +## Install a registered customer node (Windows and Linux) + +This section installs a registered Engine node connected to Agata Proxima Cloud. It is different from the disposable local development stack above. + +### Windows 10/11 and Windows Server + +**Downloads required** + +- [Download the Engine source ZIP](https://github.com/Proxima-Engine/proxima-engine/archive/refs/heads/feat/cloud-node-heartbeat-20261110.zip) and extract it. +- From Agata Proxima Cloud → Node Management, download the registered node's `.env`, one-time `node_enrollment_token`, and `tenant_signing_key`. +- The ZIP includes [install-customer.ps1](../install-customer.ps1) and [compose.customer.yaml](../compose.customer.yaml). These files must stay together in the extracted folder. + +**Prerequisites** + +Install Docker Desktop with its WSL 2 backend, enable Linux containers, and start Docker Desktop. Confirm these commands work in PowerShell: + +```powershell +docker info --format '{{.OSType}}' +docker compose version +``` + +The first command must print `linux`; Compose must report version 2.x. Proxima Engine is a Linux container, and Docker Desktop runs it on Windows through its Linux backend. + +**Prepare the downloaded files** + +1. Put the node-specific `.env` in the extracted repository root, beside `compose.customer.yaml`. +2. Create a folder named `secrets` in that same directory. +3. Put the downloaded one-time token at `secrets/node_enrollment_token`. +4. Put the tenant signing key at `secrets/tenant_signing_key`. +5. Keep the folder accessible only to the operators who administer this node. The secret filenames have no `.txt` extension; enable File name extensions in File Explorer to verify them. + +**Install** + +Open PowerShell in the extracted folder and run: + +```powershell +Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass +.\install-customer.ps1 +``` + +The installer validates the files, Docker daemon, Linux-container mode, Compose v2 configuration, starts the Engine, and checks the actual Docker container health state. It waits up to 120 seconds and prints container logs if health does not pass. On success, it prints the Engine endpoint `127.0.0.1:6432`. + +Confirm the node becomes healthy in Agata Proxima Fleet after its signed heartbeat. The Engine stores its enrolled node identity in a named Docker volume. Keep `secrets/tenant_signing_key` in place for subsequent starts; the one-time enrollment token is consumed only during first enrollment. + +**Operate** + +```powershell +docker compose -f compose.customer.yaml ps +docker compose -f compose.customer.yaml logs --tail 100 engine +docker compose -f compose.customer.yaml restart engine +docker compose -f compose.customer.yaml down +``` + +The upstream PostgreSQL hostname must be reachable from inside the Linux container. Do not set it to `localhost` unless PostgreSQL runs in that same container. Keep port 6432 private; use a private network and firewall policy when the application is on another host. + +### Linux + +Install Docker Engine and the Docker Compose v2 plugin. Place the same three node downloads and the `secrets/` files in the extracted repository root, then run: + +```bash +chmod +x ./install-customer.sh +./install-customer.sh +``` + +The Bash installer validates the files and Compose configuration, builds and starts the Engine, and waits for the Docker health check. + +### Installation scope + +The PowerShell installer installs and operates the Proxima Engine Linux container on a Windows host. It is not an MSI package or a native Windows executable/service. The reference deployment-executor service installer uses Linux systemd; executor service installation on Windows is not included in this Engine installer. See [Node enrollment](NODE_ENROLLMENT.md) and [Customer-managed deployment executor](CUSTOMER_DEPLOYMENT_EXECUTOR.md) for the complete enrollment and deployment lifecycle. From 350e89bbb4859de3e56f1cf11bfd22ddca6cb48c Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 03:55:34 -0800 Subject: [PATCH 066/108] Index Windows and Linux customer installation resources --- docs/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/README.md b/docs/README.md index e24ba7a..8dd6c55 100644 --- a/docs/README.md +++ b/docs/README.md @@ -52,3 +52,11 @@ This is the complete technical guide to the public Rust data plane. Choose a tas ## Product boundary Proxima Engine is independently buildable and testable. A compatible control plane can issue node-enrollment credentials and receive optional signed evidence. The hosted Agata Proxima Cloud application is a separate product; its private management source and hosted account workflows are not part of this repository. + +## Customer installation packages + +- [Registered-node installation guide](NODE_ENROLLMENT.md) — Windows PowerShell and Linux Bash installer steps, required downloads, enrollment credentials, Docker networking, health verification, and lifecycle operations. +- [Customer deployment executor](CUSTOMER_DEPLOYMENT_EXECUTOR.md) — signed deployment webhooks, executor credentials, approved releases, rollback, and customer-managed deployment operations. +- [Windows customer installer](../install-customer.ps1) — PowerShell installer for Windows hosts running Docker Desktop/Linux containers. +- [Linux customer installer](../install-customer.sh) — Bash installer for Linux hosts running Docker Engine and Compose v2. +- [Customer Compose definition](../compose.customer.yaml) — the container deployment and mounted secret contract. From 17fe9870d80a718ea64a528bc4652f073bba24fc Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 03:56:02 -0800 Subject: [PATCH 067/108] Fix Windows installer smoke-test command mock --- tests/windows-installer-smoke.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/windows-installer-smoke.ps1 b/tests/windows-installer-smoke.ps1 index bb8d163..6dc8c89 100644 --- a/tests/windows-installer-smoke.ps1 +++ b/tests/windows-installer-smoke.ps1 @@ -11,7 +11,7 @@ Set-Content -LiteralPath (Join-Path $tempRoot "secrets/node_enrollment_token") - Set-Content -LiteralPath (Join-Path $tempRoot "secrets/tenant_signing_key") -Value "0123456789abcdef0123456789abcdef" -NoNewline $script:dockerCalls = [System.Collections.Generic.List[string]]::new() -function global:docker { +function docker { $call = ($args | ForEach-Object { [string]$_ }) -join " " $script:dockerCalls.Add($call) $global:LASTEXITCODE = 0 From 4b2b9c4e4ea3ae5d01f8ee38bc3da75f8041b09d Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 03:56:25 -0800 Subject: [PATCH 068/108] Document Windows and Linux node installer commands --- docs/NODE_ENROLLMENT.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/NODE_ENROLLMENT.md b/docs/NODE_ENROLLMENT.md index a0c21f7..f722682 100644 --- a/docs/NODE_ENROLLMENT.md +++ b/docs/NODE_ENROLLMENT.md @@ -58,7 +58,23 @@ The Compose file mounts the enrollment token and tenant signing key as read-only When Fleet shows the node as healthy, replace the contents of secrets/node_enrollment_token with the harmless text enrollment-complete. The persisted node identity is used on subsequent starts, so the consumed one-time token is no longer needed. Keep secrets/tenant_signing_key protected and persistent. -For PowerShell, use the same files and Docker Compose commands. The host directory containing .env and secrets must be accessible only to the customer operators who manage the Engine. +For Windows, use the PowerShell installer included in the repository ZIP. Open PowerShell in the extracted repository folder and run: + +```powershell +Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass +.\\install-customer.ps1 +``` + +The installer checks that Docker Desktop is running in Linux-container mode, verifies Compose v2 and the supplied files, validates the Compose configuration, starts the Engine, and waits for the actual Docker health state. It prints the container status and recent logs if the health check fails. This is the supported Windows host installation path; the Engine itself runs as a Linux container through Docker Desktop's WSL 2 backend. It is a PowerShell installer script, not an MSI or native Windows service binary. + +For Linux, run the included Bash installer: + +```bash +chmod +x ./install-customer.sh +./install-customer.sh +``` + +The host directory containing .env and secrets must be accessible only to the customer operators who manage the Engine. Do not commit either file or the secrets directory. The Engine listens on port 6432. By default, the host port is bound to 127.0.0.1. If the application runs in another container, attach it to the shared proxima-engine-network and connect to engine:6432. Do not expose port 6432 publicly without a customer-controlled firewall rule. The upstream PostgreSQL address must be reachable from inside the Engine container; localhost means the container itself, not the host. From ad3bc3f2b51daeed5c90d59a146885def0ff8445 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 03:56:45 -0800 Subject: [PATCH 069/108] Correct Windows customer node health verification From bda86699895f1f9854d6ac799eb7241fa8a17c5f Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 03:57:00 -0800 Subject: [PATCH 070/108] Clarify Windows Engine and executor installation boundaries --- docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md index 5e2b73b..f5472ea 100644 --- a/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md +++ b/docs/CUSTOMER_DEPLOYMENT_EXECUTOR.md @@ -119,6 +119,12 @@ set +a node ./deployment-executor/receiver.mjs ``` +### Windows executor host + +The Engine customer-node installer supports Windows. The reference deployment-executor service installer in this repository is Linux/systemd-specific. On Windows, the Node.js receiver can be run interactively for development or staging with Node.js 22+, Docker Desktop in Linux-container mode, the downloaded `deployment-executor.env`, and a customer-managed HTTPS reverse proxy. It is not installed as a Windows service by `install-deployment-executor.sh`. For production automation on Windows, use a customer-managed Windows service wrapper and service account that securely injects the executor environment, or use a customer-owned orchestrator/custom executor. Do not run a production webhook receiver in an ordinary logged-in terminal and assume it will survive sign-out or reboot. + +The Engine itself is installed on Windows with [install-customer.ps1](../install-customer.ps1). That PowerShell installer is independent of the deployment-executor service installer. + ### Run as a Linux service For the standard single-host installation, place the downloaded `deployment-executor.env` in `/opt/proxima` and run the supplied installer: From 1e7a97b00fbb76dcff062424a1591e91318007fe Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 03:57:34 -0800 Subject: [PATCH 071/108] Correct Windows node health verification command --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7b43588..13036bd 100644 --- a/README.md +++ b/README.md @@ -213,10 +213,11 @@ A successful installation prints **Proxima Engine is healthy**. Then open Agata ```powershell docker compose -f compose.customer.yaml ps docker compose -f compose.customer.yaml logs --tail 100 engine -Invoke-RestMethod http://127.0.0.1:6432 +$containerId = docker compose -f compose.customer.yaml ps -q engine +docker inspect --format '{{.State.Health.Status}}' $containerId ``` -The last command is not a PostgreSQL query test; use a PostgreSQL driver or `psql` configured for `127.0.0.1:6432` to verify SQL connectivity. For the complete disposable local SQL walkthrough, follow [Quickstart](QUICKSTART.md). +The final command prints the container health state. To verify SQL connectivity, connect your application or a PostgreSQL client to `127.0.0.1:6432` using the database credentials configured for your upstream PostgreSQL service. Restart and stop the registered node: From dfa0767948c2c7bdb59d6bd8f5c742262f2c3901 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 03:57:53 -0800 Subject: [PATCH 072/108] Scope Windows installer prerequisites to supported Docker Desktop hosts --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 13036bd..9166849 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ Use [GitHub Issues](https://github.com/Agata-Proxima/proxima-engine/issues) for ## Install a registered Engine node on Windows or Linux -The customer-node installer supports Windows 10/11 and Windows Server hosts that run Docker Desktop or Docker Engine with **Linux containers** and Docker Compose v2. Windows users have a dedicated PowerShell installer; Linux users have a Bash installer. Both install the same Proxima Engine container and use the same signed enrollment and heartbeat protocol. +The customer-node installer supports Windows 10/11 hosts running Docker Desktop with the WSL 2 backend, Linux containers, and Docker Compose v2. Windows users have a dedicated PowerShell installer; Linux users have a Bash installer. Both install the same Proxima Engine container and use the same signed enrollment and heartbeat protocol. ### Windows downloads @@ -179,8 +179,7 @@ The customer-node installer supports Windows 10/11 and Windows Server hosts that ### Windows prerequisites -- Windows 10/11 or Windows Server with a supported Docker installation. -- Docker Desktop configured to use the WSL 2 backend and Linux containers, or Docker Engine with Compose v2 on the Windows host. +- Windows 10/11 with Docker Desktop configured to use the WSL 2 backend and Linux containers. - Docker daemon running before installation. - The node-specific `.env` and both secret files from Node Management. - Network access from the Engine container to the configured PostgreSQL upstream and Agata Proxima HTTPS enrollment/heartbeat endpoints. From 8749f2ddaff0a0b65712772d90abd3d185676d45 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 03:58:01 -0800 Subject: [PATCH 073/108] Scope Windows installer prerequisites to supported Docker Desktop hosts --- docs/QUICKSTART.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 963f6f7..a5bdf56 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -147,7 +147,7 @@ Connect your PostgreSQL client to port 6432. The Engine reads configuration from This section installs a registered Engine node connected to Agata Proxima Cloud. It is different from the disposable local development stack above. -### Windows 10/11 and Windows Server +### Windows 10/11 **Downloads required** From f6b3d0b359ad5dda8d16dacbc0c934c3083fd51f Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 03:58:54 -0800 Subject: [PATCH 074/108] Fix Windows CI Docker mock scope --- tests/windows-installer-smoke.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/windows-installer-smoke.ps1 b/tests/windows-installer-smoke.ps1 index 6dc8c89..2134137 100644 --- a/tests/windows-installer-smoke.ps1 +++ b/tests/windows-installer-smoke.ps1 @@ -10,10 +10,10 @@ Set-Content -LiteralPath (Join-Path $tempRoot ".env") -Value ("PROXIMA_NODE_ID=0 Set-Content -LiteralPath (Join-Path $tempRoot "secrets/node_enrollment_token") -Value "test-one-time-token" -NoNewline Set-Content -LiteralPath (Join-Path $tempRoot "secrets/tenant_signing_key") -Value "0123456789abcdef0123456789abcdef" -NoNewline -$script:dockerCalls = [System.Collections.Generic.List[string]]::new() -function docker { +$global:dockerCalls = [System.Collections.Generic.List[string]]::new() +function global:docker { $call = ($args | ForEach-Object { [string]$_ }) -join " " - $script:dockerCalls.Add($call) + $global:dockerCalls.Add($call) $global:LASTEXITCODE = 0 if ($call -eq "info --format {{.OSType}}") { return "linux" } @@ -38,7 +38,7 @@ try { "inspect --format {{.State.Health.Status}} test-engine-container" ) foreach ($requiredCall in $requiredCalls) { - if (-not $script:dockerCalls.Contains($requiredCall)) { + if (-not $global:dockerCalls.Contains($requiredCall)) { throw "Windows installer did not perform required Docker operation: $requiredCall" } } From 5e43a255358e1eaaecc723384e80bbdf148a026c Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:26:22 -0800 Subject: [PATCH 075/108] Use stable main-branch customer installer download links --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9166849..0d2fd44 100644 --- a/README.md +++ b/README.md @@ -169,9 +169,9 @@ The customer-node installer supports Windows 10/11 hosts running Docker Desktop ### Windows downloads -1. Download the [Proxima Engine source ZIP](https://github.com/Proxima-Engine/proxima-engine/archive/refs/heads/feat/cloud-node-heartbeat-20261110.zip) and extract it. +1. Download the [Proxima Engine source ZIP](https://github.com/Proxima-Engine/proxima-engine/archive/refs/heads/main.zip) and extract it. 2. Download the node-specific `.env`, `node_enrollment_token`, and `tenant_signing_key` files from Agata Proxima Cloud → Node Management → the registered node's setup/download section. These are generated credentials; they are not public repository files. -3. Download the [Windows PowerShell installer](https://raw.githubusercontent.com/Proxima-Engine/proxima-engine/feat/cloud-node-heartbeat-20261110/install-customer.ps1) if it is not already present in the ZIP. The [customer Compose file](https://raw.githubusercontent.com/Proxima-Engine/proxima-engine/feat/cloud-node-heartbeat-20261110/compose.customer.yaml) must remain beside it. +3. Download the [Windows PowerShell installer](https://raw.githubusercontent.com/Proxima-Engine/proxima-engine/main/install-customer.ps1) if it is not already present in the ZIP. The [customer Compose file](https://raw.githubusercontent.com/Proxima-Engine/proxima-engine/main/compose.customer.yaml) must remain beside it. 4. In the extracted folder, create a directory named `secrets`. Place the downloaded secret files there using the exact extensionless filenames: - `secrets/node_enrollment_token` - `secrets/tenant_signing_key` From 1847ac0fc848c7460110981c0d422526e64f7b26 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:26:25 -0800 Subject: [PATCH 076/108] Use stable main-branch customer installer download links --- docs/QUICKSTART.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index a5bdf56..63f0e78 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -151,7 +151,7 @@ This section installs a registered Engine node connected to Agata Proxima Cloud. **Downloads required** -- [Download the Engine source ZIP](https://github.com/Proxima-Engine/proxima-engine/archive/refs/heads/feat/cloud-node-heartbeat-20261110.zip) and extract it. +- [Download the Engine source ZIP](https://github.com/Proxima-Engine/proxima-engine/archive/refs/heads/main.zip) and extract it. - From Agata Proxima Cloud → Node Management, download the registered node's `.env`, one-time `node_enrollment_token`, and `tenant_signing_key`. - The ZIP includes [install-customer.ps1](../install-customer.ps1) and [compose.customer.yaml](../compose.customer.yaml). These files must stay together in the extracted folder. From 6a50b870e1a3ba7dccfcdd7a84cc09acfd68dda2 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:46:34 -0800 Subject: [PATCH 077/108] Load organization-scoped tenant key from enrolled node identity --- crates/proxima-engine/src/main.rs | 53 ++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/crates/proxima-engine/src/main.rs b/crates/proxima-engine/src/main.rs index f5a4899..c201e6b 100644 --- a/crates/proxima-engine/src/main.rs +++ b/crates/proxima-engine/src/main.rs @@ -224,12 +224,33 @@ async fn bootstrap_node_identity() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Date: Tue, 10 Nov 2026 22:47:09 -0800 Subject: [PATCH 078/108] Use the organization-scoped tenant key delivered in node identity --- compose.customer.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/compose.customer.yaml b/compose.customer.yaml index 7ede85d..174fa35 100644 --- a/compose.customer.yaml +++ b/compose.customer.yaml @@ -11,7 +11,7 @@ services: PROXIMA_NODE_ENROLLMENT_URL: ${PROXIMA_NODE_ENROLLMENT_URL:-https://api.agataproxima.com/api/v1/nodes/enroll} PROXIMA_NODE_HEARTBEAT_URL: ${PROXIMA_NODE_HEARTBEAT_URL:-https://api.agataproxima.com/api/v1/nodes/heartbeat} PROXIMA_NODE_ENROLLMENT_TOKEN_FILE: /run/secrets/node_enrollment_token - PROXIMA_TENANT_SIGNING_KEY_FILE: /run/secrets/tenant_signing_key + PROXIMA_ENVIRONMENT: ${PROXIMA_ENVIRONMENT:-production} PROXIMA_LISTEN_ADDR: ${PROXIMA_LISTEN_ADDR:-0.0.0.0:6432} PROXIMA_UPSTREAM_ADDR: ${PROXIMA_UPSTREAM_ADDR:?Set PROXIMA_UPSTREAM_ADDR in .env} PROXIMA_DASHBOARD_ENABLED: "false" @@ -20,7 +20,6 @@ services: - "${PROXIMA_BIND_ADDRESS:-127.0.0.1}:6432:6432" secrets: - node_enrollment_token - - tenant_signing_key volumes: - proxima-node-data:/var/lib/proxima healthcheck: @@ -38,8 +37,6 @@ services: secrets: node_enrollment_token: file: ./secrets/node_enrollment_token - tenant_signing_key: - file: ./secrets/tenant_signing_key networks: default: From 3ca282a0bd45d45d4b57319a6f9459494b464957 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:47:12 -0800 Subject: [PATCH 079/108] Make Windows customer installer create its environment and enrollment secret --- install-customer.ps1 | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/install-customer.ps1 b/install-customer.ps1 index 1805df0..f702a31 100644 --- a/install-customer.ps1 +++ b/install-customer.ps1 @@ -4,17 +4,46 @@ Set-StrictMode -Version Latest $root = Split-Path -Parent $MyInvocation.MyCommand.Path Set-Location $root -$required = @(".env", "compose.customer.yaml", "secrets/node_enrollment_token", "secrets/tenant_signing_key") -foreach ($path in $required) { - if (-not (Test-Path -LiteralPath (Join-Path $root $path) -PathType Leaf)) { - throw "Missing required customer setup file: $path. Download .env and both secret files from Agata Proxima → Nodes, then place them in this folder." - } +if (-not (Test-Path -LiteralPath (Join-Path $root "compose.customer.yaml") -PathType Leaf)) { + throw "compose.customer.yaml is missing. Run this installer from the extracted Proxima Engine repository folder." } if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { throw "Docker CLI was not found. Install Docker Desktop with the WSL 2 backend and Docker Compose v2, then rerun this script." } +if (-not (Test-Path -LiteralPath (Join-Path $root ".env") -PathType Leaf)) { + if ([string]::IsNullOrWhiteSpace($env:PROXIMA_NODE_ID)) { + throw "Set PROXIMA_NODE_ID using the enrollment configuration copied from Agata Proxima → Nodes." + } + if ([string]::IsNullOrWhiteSpace($env:PROXIMA_NODE_ENROLLMENT_URL)) { + $env:PROXIMA_NODE_ENROLLMENT_URL = "https://api.agataproxima.com/api/v1/nodes/enroll" + } + if ([string]::IsNullOrWhiteSpace($env:PROXIMA_UPSTREAM_ADDR)) { + $env:PROXIMA_UPSTREAM_ADDR = Read-Host "PostgreSQL upstream host:port (for example, db.example.com:5432)" + } + if ([string]::IsNullOrWhiteSpace($env:PROXIMA_UPSTREAM_ADDR)) { + throw "A PostgreSQL upstream address is required." + } + @( + "PROXIMA_NODE_ID=$($env:PROXIMA_NODE_ID)" + "PROXIMA_NODE_ENROLLMENT_URL=$($env:PROXIMA_NODE_ENROLLMENT_URL)" + "PROXIMA_UPSTREAM_ADDR=$($env:PROXIMA_UPSTREAM_ADDR)" + "PROXIMA_LISTEN_ADDR=$($env:PROXIMA_LISTEN_ADDR ?? '0.0.0.0:6432')" + "PROXIMA_ENVIRONMENT=production" + ) | Set-Content -LiteralPath (Join-Path $root ".env") -Encoding ascii +} + +$tokenPath = Join-Path $root "secrets/node_enrollment_token" +if (-not (Test-Path -LiteralPath $tokenPath -PathType Leaf)) { + if ([string]::IsNullOrWhiteSpace($env:PROXIMA_NODE_ENROLLMENT_TOKEN)) { + throw "The one-time enrollment token is missing. Copy the enrollment configuration from Agata Proxima → Nodes and rerun this installer." + } + New-Item -ItemType Directory -Force -Path (Split-Path -Parent $tokenPath) | Out-Null + [System.IO.File]::WriteAllText($tokenPath, $env:PROXIMA_NODE_ENROLLMENT_TOKEN, [System.Text.Encoding]::UTF8) +} +Remove-Item Env:PROXIMA_NODE_ENROLLMENT_TOKEN -ErrorAction SilentlyContinue + $dockerOs = docker info --format '{{.OSType}}' if ($LASTEXITCODE -ne 0) { throw "Docker Desktop is not running or the Docker daemon is unavailable. Start Docker Desktop, wait until it reports Engine running, and rerun this script." @@ -29,7 +58,7 @@ Write-Host $version $configOutput = docker compose -f compose.customer.yaml config --quiet 2>&1 if ($LASTEXITCODE -ne 0) { - throw "Customer Compose configuration is invalid. Check .env and the two secret-file paths. Details: $configOutput" + throw "Customer Compose configuration is invalid. Check the generated .env file and enrollment-token secret. Details: $configOutput" } Write-Host "Building and starting Proxima Engine. The first build downloads Rust dependencies and can take several minutes." @@ -65,6 +94,6 @@ if (-not $healthy) { Write-Host "" Write-Host "Proxima Engine is healthy." Write-Host "PostgreSQL proxy: 127.0.0.1:6432" -Write-Host "Node enrollment and signed heartbeat now run from the Engine container." +Write-Host "Node enrollment and signed heartbeat are active. The organization-scoped tenant verification key is stored in the protected node identity file." Write-Host "Confirm that the registered node appears as healthy in Agata Proxima Fleet." docker compose -f compose.customer.yaml ps From c8f74b7ee6121049a5cec8ffce3654e21ac4e2ee Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:47:14 -0800 Subject: [PATCH 080/108] Make Linux customer installer create its environment and enrollment secret --- install-customer.sh | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/install-customer.sh b/install-customer.sh index ce2e3b5..d45f34f 100644 --- a/install-customer.sh +++ b/install-customer.sh @@ -2,15 +2,35 @@ set -Eeuo pipefail cd "$(dirname "$0")" -for file in .env compose.customer.yaml secrets/node_enrollment_token secrets/tenant_signing_key; do - if [[ ! -f "$file" ]]; then - echo "Missing $file. Download .env and both secret files from Agata Proxima → Nodes, then place them in this folder." >&2 - exit 1 - fi -done +[[ -f compose.customer.yaml ]] || { echo "Run this installer from the extracted Proxima Engine repository folder." >&2; exit 1; } command -v docker >/dev/null 2>&1 || { echo "Docker CLI is required." >&2; exit 1; } -docker compose version >/dev/null +if [[ ! -f .env ]]; then + : "${PROXIMA_NODE_ID:?Set PROXIMA_NODE_ID using the enrollment configuration copied from Agata Proxima → Nodes}" + PROXIMA_NODE_ENROLLMENT_URL="${PROXIMA_NODE_ENROLLMENT_URL:-https://api.agataproxima.com/api/v1/nodes/enroll}" + if [[ -z "${PROXIMA_UPSTREAM_ADDR:-}" ]]; then + read -r -p "PostgreSQL upstream host:port (for example, db.example.com:5432): " PROXIMA_UPSTREAM_ADDR + fi + [[ -n "$PROXIMA_UPSTREAM_ADDR" ]] || { echo "A PostgreSQL upstream address is required." >&2; exit 1; } + cat > .env < "$token_file" +fi +unset PROXIMA_NODE_ENROLLMENT_TOKEN || true + +docker compose version >/dev/null docker compose -f compose.customer.yaml config --quiet echo "Building and starting Proxima Engine. The first build downloads Rust dependencies and can take several minutes." docker compose -f compose.customer.yaml up -d --build @@ -19,7 +39,7 @@ echo "Waiting for Engine health check..." for attempt in $(seq 1 40); do if docker compose -f compose.customer.yaml ps --format json engine | grep -q '"Health":"healthy"'; then docker compose -f compose.customer.yaml ps - echo "Proxima Engine is healthy. Node enrollment and signed heartbeat will now complete against the configured Agata Proxima Control Plane." + echo "Proxima Engine is healthy. Node enrollment and signed heartbeat are active. The organization-scoped tenant verification key is stored in the protected node identity file." exit 0 fi sleep 3 From 7d3fb73828767d640f167c4cc57a452f0eed6a1d Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:47:25 -0800 Subject: [PATCH 081/108] Keep the Windows installer compatible with Windows PowerShell 5.1 --- install-customer.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/install-customer.ps1 b/install-customer.ps1 index f702a31..bd2c005 100644 --- a/install-customer.ps1 +++ b/install-customer.ps1 @@ -22,6 +22,9 @@ if (-not (Test-Path -LiteralPath (Join-Path $root ".env") -PathType Leaf)) { if ([string]::IsNullOrWhiteSpace($env:PROXIMA_UPSTREAM_ADDR)) { $env:PROXIMA_UPSTREAM_ADDR = Read-Host "PostgreSQL upstream host:port (for example, db.example.com:5432)" } + if ([string]::IsNullOrWhiteSpace($env:PROXIMA_LISTEN_ADDR)) { + $env:PROXIMA_LISTEN_ADDR = "0.0.0.0:6432" + } if ([string]::IsNullOrWhiteSpace($env:PROXIMA_UPSTREAM_ADDR)) { throw "A PostgreSQL upstream address is required." } @@ -29,7 +32,7 @@ if (-not (Test-Path -LiteralPath (Join-Path $root ".env") -PathType Leaf)) { "PROXIMA_NODE_ID=$($env:PROXIMA_NODE_ID)" "PROXIMA_NODE_ENROLLMENT_URL=$($env:PROXIMA_NODE_ENROLLMENT_URL)" "PROXIMA_UPSTREAM_ADDR=$($env:PROXIMA_UPSTREAM_ADDR)" - "PROXIMA_LISTEN_ADDR=$($env:PROXIMA_LISTEN_ADDR ?? '0.0.0.0:6432')" + "PROXIMA_LISTEN_ADDR=$($env:PROXIMA_LISTEN_ADDR)" "PROXIMA_ENVIRONMENT=production" ) | Set-Content -LiteralPath (Join-Path $root ".env") -Encoding ascii } From 81d8b5ee9feac3534be65fa936723cadb8e704b7 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:47:52 -0800 Subject: [PATCH 082/108] Write enrollment token without a UTF-8 BOM --- install-customer.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-customer.ps1 b/install-customer.ps1 index bd2c005..44165ea 100644 --- a/install-customer.ps1 +++ b/install-customer.ps1 @@ -43,7 +43,7 @@ if (-not (Test-Path -LiteralPath $tokenPath -PathType Leaf)) { throw "The one-time enrollment token is missing. Copy the enrollment configuration from Agata Proxima → Nodes and rerun this installer." } New-Item -ItemType Directory -Force -Path (Split-Path -Parent $tokenPath) | Out-Null - [System.IO.File]::WriteAllText($tokenPath, $env:PROXIMA_NODE_ENROLLMENT_TOKEN, [System.Text.Encoding]::UTF8) + [System.IO.File]::WriteAllText($tokenPath, $env:PROXIMA_NODE_ENROLLMENT_TOKEN, [System.Text.UTF8Encoding]::new($false)) } Remove-Item Env:PROXIMA_NODE_ENROLLMENT_TOKEN -ErrorAction SilentlyContinue From 26fd60cc5b29a33034134964611d644675aab50e Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:47:54 -0800 Subject: [PATCH 083/108] Validate customer Compose with only the enrollment secret and generated environment --- .github/workflows/ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0158a05..a6f5714 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,8 +51,14 @@ jobs: run: | mkdir -p secrets printf 'test-enrollment-token' > secrets/node_enrollment_token - printf '0123456789abcdef0123456789abcdef' > secrets/tenant_signing_key - PROXIMA_NODE_ID=00000000-0000-4000-8000-000000000001 PROXIMA_UPSTREAM_ADDR=127.0.0.1:5432 docker compose -f compose.customer.yaml config --quiet + cat > .env <<'EOF' + PROXIMA_NODE_ID=00000000-0000-4000-8000-000000000001 + PROXIMA_NODE_ENROLLMENT_URL=https://api.agataproxima.com/api/v1/nodes/enroll + PROXIMA_UPSTREAM_ADDR=127.0.0.1:5432 + PROXIMA_LISTEN_ADDR=0.0.0.0:6432 + PROXIMA_ENVIRONMENT=production + EOF + docker compose -f compose.customer.yaml config --quiet - name: Run engine tests run: cargo test -p proxima-engine - name: Build release engine From c55cd5b2058aa95567b023ed2b91b6f21279468f Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:47:57 -0800 Subject: [PATCH 084/108] Exercise Windows installer environment and one-time secret generation --- tests/windows-installer-smoke.ps1 | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/windows-installer-smoke.ps1 b/tests/windows-installer-smoke.ps1 index 2134137..045d3d1 100644 --- a/tests/windows-installer-smoke.ps1 +++ b/tests/windows-installer-smoke.ps1 @@ -3,12 +3,13 @@ Set-StrictMode -Version Latest $sourceRoot = Split-Path -Parent (Split-Path -Parent $MyInvocation.MyCommand.Path) $tempRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("proxima-windows-installer-test-" + [guid]::NewGuid().ToString("N")) -New-Item -ItemType Directory -Path (Join-Path $tempRoot "secrets") -Force | Out-Null +New-Item -ItemType Directory -Path $tempRoot -Force | Out-Null Copy-Item (Join-Path $sourceRoot "install-customer.ps1") $tempRoot Copy-Item (Join-Path $sourceRoot "compose.customer.yaml") $tempRoot -Set-Content -LiteralPath (Join-Path $tempRoot ".env") -Value ("PROXIMA_NODE_ID=00000000-0000-4000-8000-000000000001" + [Environment]::NewLine + "PROXIMA_UPSTREAM_ADDR=postgres:5432") -NoNewline -Set-Content -LiteralPath (Join-Path $tempRoot "secrets/node_enrollment_token") -Value "test-one-time-token" -NoNewline -Set-Content -LiteralPath (Join-Path $tempRoot "secrets/tenant_signing_key") -Value "0123456789abcdef0123456789abcdef" -NoNewline +$env:PROXIMA_NODE_ID = "00000000-0000-4000-8000-000000000001" +$env:PROXIMA_NODE_ENROLLMENT_URL = "https://api.agataproxima.com/api/v1/nodes/enroll" +$env:PROXIMA_NODE_ENROLLMENT_TOKEN = "test-one-time-token" +$env:PROXIMA_UPSTREAM_ADDR = "postgres:5432" $global:dockerCalls = [System.Collections.Generic.List[string]]::new() function global:docker { @@ -29,6 +30,18 @@ function global:docker { try { & (Join-Path $tempRoot "install-customer.ps1") + $generatedEnv = Get-Content -LiteralPath (Join-Path $tempRoot ".env") -Raw + $generatedToken = Get-Content -LiteralPath (Join-Path $tempRoot "secrets/node_enrollment_token") -Raw + if ($generatedEnv -notmatch "PROXIMA_UPSTREAM_ADDR=postgres:5432") { + throw "Windows installer did not generate the PostgreSQL upstream setting." + } + if ($generatedToken.Trim() -ne "test-one-time-token") { + throw "Windows installer did not create the one-time enrollment secret file correctly." + } + if (Test-Path Env:PROXIMA_NODE_ENROLLMENT_TOKEN) { + throw "Windows installer did not remove the one-time token from the current process environment." + } + $requiredCalls = @( "info --format {{.OSType}}", "compose version", From 50500cbcdecfe729bce61ef51b930ec6c46ed07b Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:51:34 -0800 Subject: [PATCH 085/108] Rewrite customer node install guide around secure automatic key delivery --- docs/NODE_ENROLLMENT.md | 43 +++++++++++++---------------------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/docs/NODE_ENROLLMENT.md b/docs/NODE_ENROLLMENT.md index f722682..0136744 100644 --- a/docs/NODE_ENROLLMENT.md +++ b/docs/NODE_ENROLLMENT.md @@ -41,42 +41,25 @@ Use HTTPS for remote control-plane hosts. Loopback HTTP is reserved for local de ## 2. One-command Docker installation (hosted Cloud) -This is the recommended customer path. Docker Engine and Docker Compose v2 are required on the customer's host. Agata Proxima does not provision, own, or receive the customer's PostgreSQL credentials. +This is the supported customer path. Docker Engine and Docker Compose v2 are required on Linux. Windows 10/11 hosts use Docker Desktop with its WSL 2 backend and Linux containers. Agata Proxima does not provision, own, or receive the customer's PostgreSQL credentials. -1. Register a node in Agata Proxima Cloud and download the three generated files from the registration result: .env, node_enrollment_token, and tenant_signing_key. -2. Download the public Engine repository ZIP and extract it on the customer host. -3. Place .env and compose.customer.yaml in the repository root. Create a secrets directory beside them and save the two secret downloads as secrets/node_enrollment_token and secrets/tenant_signing_key. -4. Set the upstream address in .env to the PostgreSQL host reachable from the Engine container. The tenant signing key must be the same secret the trusted application backend uses to sign tenant context. Never reuse the one-time enrollment token as the tenant signing key. -5. Start the Engine: +1. Register a node in Agata Proxima Cloud → Nodes. Copy the Windows PowerShell setup or Linux shell setup shown with the one-time enrollment token. +2. Download the public Proxima Engine source ZIP and extract it on the customer host. +3. Open a terminal in the extracted repository folder, paste the matching setup commands into that same terminal session, and run the installer: + - Windows: `Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass` followed by `./install-customer.ps1` + - Linux: `chmod +x ./install-customer.sh` followed by `./install-customer.sh` +4. When prompted, enter the PostgreSQL upstream host and port reachable from inside the Engine container. The installer writes the node's `.env` file and the one-time enrollment token file automatically. +5. Confirm the installer reports a healthy Engine container, then confirm the node reports healthy in Agata Proxima Fleet after its signed heartbeat. -```bash -docker compose -f compose.customer.yaml up --build -d -docker compose -f compose.customer.yaml logs -f engine -``` - -The Compose file mounts the enrollment token and tenant signing key as read-only secret files. Neither secret is passed as a container environment variable. The Engine runs as UID 10001. On Linux, make the secret directory and files readable by that UID while keeping them private from other host users (for example, chown the directory and files to 10001:10001, chmod the directory 0700 and the files 0400). On Docker Desktop, confirm the mounted files are readable inside the Linux container; do not make the host secrets directory broadly readable just to work around a permission issue. The Engine reads the enrollment token file only during first enrollment, persists its node identity in the named Docker volume, and sends signed heartbeats every 30 seconds. No heartbeat cron job, custom webhook receiver, or customer-written enrollment code is required. +The customer Compose file mounts only the one-time enrollment token as a read-only secret. During the HTTPS enrollment exchange, the Control Plane returns the generated node heartbeat key and a separate organization-scoped tenant-token verification key. The Engine validates both and stores them in the protected node identity file. The Control Plane master signing secret is never sent to customer nodes. The identity file is persisted in a named Docker volume; on Unix, it is created with owner-only permissions. No customer has to download or manually copy a tenant signing key. -When Fleet shows the node as healthy, replace the contents of secrets/node_enrollment_token with the harmless text enrollment-complete. The persisted node identity is used on subsequent starts, so the consumed one-time token is no longer needed. Keep secrets/tenant_signing_key protected and persistent. +The enrollment token is consumed during the first bootstrap. After Fleet shows the node as healthy, replace the contents of `secrets/node_enrollment_token` with the harmless text `enrollment-complete`. Keep the file itself in place because Compose references it on subsequent starts. The persisted identity is used on every later start. -For Windows, use the PowerShell installer included in the repository ZIP. Open PowerShell in the extracted repository folder and run: - -```powershell -Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -.\\install-customer.ps1 -``` - -The installer checks that Docker Desktop is running in Linux-container mode, verifies Compose v2 and the supplied files, validates the Compose configuration, starts the Engine, and waits for the actual Docker health state. It prints the container status and recent logs if the health check fails. This is the supported Windows host installation path; the Engine itself runs as a Linux container through Docker Desktop's WSL 2 backend. It is a PowerShell installer script, not an MSI or native Windows service binary. - -For Linux, run the included Bash installer: - -```bash -chmod +x ./install-customer.sh -./install-customer.sh -``` +The Windows PowerShell installer checks Docker Desktop's Linux-container mode, Compose v2, the generated environment and token file, then builds and starts the Engine and waits for the actual Docker health state. It is a Windows host installer for the Linux Engine container, not an MSI or a native Windows service binary. -The host directory containing .env and secrets must be accessible only to the customer operators who manage the Engine. Do not commit either file or the secrets directory. +The Linux Bash installer performs the same setup and health checks. Keep the repository's `.env` and `secrets/` directory accessible only to the operators who administer the node. Do not commit them. -The Engine listens on port 6432. By default, the host port is bound to 127.0.0.1. If the application runs in another container, attach it to the shared proxima-engine-network and connect to engine:6432. Do not expose port 6432 publicly without a customer-controlled firewall rule. The upstream PostgreSQL address must be reachable from inside the Engine container; localhost means the container itself, not the host. +The Engine listens on port 6432. By default, the host port is bound to 127.0.0.1. If the application runs in another container, attach it to the shared `proxima-engine-network` and connect to `engine:6432`. Do not expose port 6432 publicly without a customer-controlled firewall rule. The upstream PostgreSQL address must be reachable from inside the Engine container; localhost means the container itself, not the host. ## 3. Configure the Engine — PowerShell From e379a5626794e6ecf89617540d7b5308c60fa962 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:51:43 -0800 Subject: [PATCH 086/108] Document organization-scoped tenant key persistence and credential handling --- docs/NODE_ENROLLMENT.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/NODE_ENROLLMENT.md b/docs/NODE_ENROLLMENT.md index 0136744..0114f84 100644 --- a/docs/NODE_ENROLLMENT.md +++ b/docs/NODE_ENROLLMENT.md @@ -76,7 +76,7 @@ $env:PROXIMA_UPSTREAM_ADDR = "127.0.0.1:5432" cargo run -p proxima-engine ``` -The Engine sends `POST /api/v1/nodes/enroll` with JSON fields `node_id` and `enrollment_token`. The control plane validates the one-time token and returns the node identity and signing credential. The Engine validates the response and persists the identity. +The Engine sends `POST /api/v1/nodes/enroll` with JSON fields `node_id` and `enrollment_token`. The Control Plane validates the one-time token and returns the node heartbeat key plus the organization's derived tenant-token verification key. The Engine validates both credentials and persists them in the protected identity file. The Control Plane master secret is never returned. Remove the bootstrap token after enrollment: @@ -105,7 +105,7 @@ Default identity paths: - Windows: `%LOCALAPPDATA%\AgataProxima\node-identity.json`, when `LOCALAPPDATA` exists. - Other environments: `data/node-identity.json`. -Set `PROXIMA_NODE_IDENTITY_PATH` to choose another path. The identity file contains the node ID, organization ID, and a 32-byte signing credential. Restrict file permissions and include the file in protected backups. +Set `PROXIMA_NODE_IDENTITY_PATH` to choose another path. The identity file contains the node ID, organization ID, the 32-byte node heartbeat key and the 32-byte organization-scoped tenant-token verification key. Restrict file permissions and include the file in protected backups. The Control Plane master secret is not stored on the Engine. The Engine reuses a valid saved identity on later starts, so the one-time enrollment token is needed only for the initial bootstrap. Use a separate identity path for each node. @@ -118,10 +118,10 @@ When the Engine has a registered node identity and a compatible heartbeat endpoi | `PROXIMA_NODE_ENROLLMENT_TOKEN` | One-time bootstrap credential | Short-lived; remove after enrollment | | `PROXIMA_NODE_IDENTITY_PATH` | Persistent identity file | Contains a signing credential; protect it | | `PROXIMA_NODE_SIGNING_KEY_HEX` | 32-byte node signing key, hex encoded | Normally loaded from saved identity; never commit | -| `PROXIMA_TENANT_SIGNING_KEY` | HMAC key for application-signed tenant context | Separate from node credentials; at least 32 bytes | +| `PROXIMA_TENANT_SIGNING_KEY` | Organization-scoped HMAC key for tenant context | Automatically loaded from the enrolled identity on Cloud-registered nodes; for manual provisioning, configure the trusted signer's matching key. Never copy the Control Plane master secret to a customer node | | `PROXIMA_ATTESTATION_INGEST_TOKEN` | Optional bearer credential for a compatible evidence receiver | Use the required `aga_` prefix; never commit | -These credentials serve different trust boundaries. Keep each in its designated configuration field and rotate it through the corresponding issuer. +These credentials serve different trust boundaries. The Control Plane master secret derives organization-specific tenant keys; the node identity carries only the key for its own organization. Keep each credential in its designated secret store and rotate it through the corresponding issuer. ## Troubleshooting From 60c9b1f9bdd4a2e43b913c5eea13a2f684c99a6e Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:51:55 -0800 Subject: [PATCH 087/108] Simplify Windows and Linux registered-node quickstart --- docs/QUICKSTART.md | 61 ++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 63f0e78..33eceef 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -147,67 +147,58 @@ Connect your PostgreSQL client to port 6432. The Engine reads configuration from This section installs a registered Engine node connected to Agata Proxima Cloud. It is different from the disposable local development stack above. -### Windows 10/11 - -**Downloads required** +### 1. Register the node -- [Download the Engine source ZIP](https://github.com/Proxima-Engine/proxima-engine/archive/refs/heads/main.zip) and extract it. -- From Agata Proxima Cloud → Node Management, download the registered node's `.env`, one-time `node_enrollment_token`, and `tenant_signing_key`. -- The ZIP includes [install-customer.ps1](../install-customer.ps1) and [compose.customer.yaml](../compose.customer.yaml). These files must stay together in the extracted folder. +In Agata Proxima Cloud → Nodes, create the node in the intended organization. Copy the Windows PowerShell setup or Linux shell setup shown with the one-time enrollment token. The token is single-use and is not the tenant signing key. -**Prerequisites** +### 2. Download the Engine installer -Install Docker Desktop with its WSL 2 backend, enable Linux containers, and start Docker Desktop. Confirm these commands work in PowerShell: - -```powershell -docker info --format '{{.OSType}}' -docker compose version -``` +Download the [Proxima Engine source ZIP](https://github.com/Proxima-Engine/proxima-engine/archive/refs/heads/main.zip) and extract it. The ZIP includes the Windows PowerShell installer, Linux Bash installer and customer Compose file. -The first command must print `linux`; Compose must report version 2.x. Proxima Engine is a Linux container, and Docker Desktop runs it on Windows through its Linux backend. +### Windows 10/11 -**Prepare the downloaded files** +**Prerequisites** -1. Put the node-specific `.env` in the extracted repository root, beside `compose.customer.yaml`. -2. Create a folder named `secrets` in that same directory. -3. Put the downloaded one-time token at `secrets/node_enrollment_token`. -4. Put the tenant signing key at `secrets/tenant_signing_key`. -5. Keep the folder accessible only to the operators who administer this node. The secret filenames have no `.txt` extension; enable File name extensions in File Explorer to verify them. +- Docker Desktop installed and running with its WSL 2 backend. +- Docker Desktop switched to Linux containers. +- Network access from the Engine container to the PostgreSQL upstream and Agata Proxima HTTPS enrollment/heartbeat endpoints. **Install** -Open PowerShell in the extracted folder and run: +Open PowerShell in the extracted Engine repository folder. Paste the Windows setup commands copied from Agata Proxima → Nodes into the same PowerShell session. Then run: ```powershell Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass .\install-customer.ps1 ``` -The installer validates the files, Docker daemon, Linux-container mode, Compose v2 configuration, starts the Engine, and checks the actual Docker container health state. It waits up to 120 seconds and prints container logs if health does not pass. On success, it prints the Engine endpoint `127.0.0.1:6432`. +The installer creates `.env` and `secrets/node_enrollment_token` from the copied node configuration. It asks for the PostgreSQL upstream host and port if `PROXIMA_UPSTREAM_ADDR` is not already set. It validates Docker Desktop's Linux-container mode and Compose v2, validates the Compose configuration, builds and starts the Engine, and waits up to 120 seconds for the actual Docker health state. On success, it prints the local PostgreSQL proxy endpoint `127.0.0.1:6432`. -Confirm the node becomes healthy in Agata Proxima Fleet after its signed heartbeat. The Engine stores its enrolled node identity in a named Docker volume. Keep `secrets/tenant_signing_key` in place for subsequent starts; the one-time enrollment token is consumed only during first enrollment. - -**Operate** +### Linux -```powershell -docker compose -f compose.customer.yaml ps -docker compose -f compose.customer.yaml logs --tail 100 engine -docker compose -f compose.customer.yaml restart engine -docker compose -f compose.customer.yaml down -``` +**Prerequisites** -The upstream PostgreSQL hostname must be reachable from inside the Linux container. Do not set it to `localhost` unless PostgreSQL runs in that same container. Keep port 6432 private; use a private network and firewall policy when the application is on another host. +- Docker Engine and Docker Compose v2. +- Network access from the Engine container to the PostgreSQL upstream and Agata Proxima HTTPS enrollment/heartbeat endpoints. -### Linux +**Install** -Install Docker Engine and the Docker Compose v2 plugin. Place the same three node downloads and the `secrets/` files in the extracted repository root, then run: +Open a shell in the extracted Engine repository folder. Paste the Linux setup commands copied from Agata Proxima → Nodes into that same shell session. Then run: ```bash chmod +x ./install-customer.sh ./install-customer.sh ``` -The Bash installer validates the files and Compose configuration, builds and starts the Engine, and waits for the Docker health check. +The Bash installer creates `.env` and the one-time enrollment token file, prompts for the PostgreSQL upstream when needed, validates Compose, builds and starts the Engine, and waits for the Docker health check. + +### After installation + +During first enrollment, the Control Plane returns the node heartbeat key and a separate organization-scoped tenant-token verification key over HTTPS. The Engine validates both and stores them in the protected node identity file in a named Docker volume. The Control Plane master signing secret is never sent to customer nodes. No customer has to download, create or manually copy a tenant signing key. + +Confirm the node reports healthy in Agata Proxima Fleet after its signed heartbeat. The one-time enrollment token is consumed on first bootstrap. Replace the contents of `secrets/node_enrollment_token` with the harmless text `enrollment-complete`; keep the file itself in place because Compose references it on later starts. + +The Engine listens on port 6432 and binds to host loopback by default. The upstream PostgreSQL hostname must be reachable from inside the Engine container. Do not set it to `localhost` unless PostgreSQL runs in that same container. Keep port 6432 private; use a private network and firewall policy when the application is on another host. ### Installation scope From 09566c71f9c7987e27f3f9b5ea85d434be1f1c7f Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:52:17 -0800 Subject: [PATCH 088/108] Correct customer installer and tenant-key security instructions --- README.md | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 0d2fd44..dc49959 100644 --- a/README.md +++ b/README.md @@ -170,18 +170,15 @@ The customer-node installer supports Windows 10/11 hosts running Docker Desktop ### Windows downloads 1. Download the [Proxima Engine source ZIP](https://github.com/Proxima-Engine/proxima-engine/archive/refs/heads/main.zip) and extract it. -2. Download the node-specific `.env`, `node_enrollment_token`, and `tenant_signing_key` files from Agata Proxima Cloud → Node Management → the registered node's setup/download section. These are generated credentials; they are not public repository files. -3. Download the [Windows PowerShell installer](https://raw.githubusercontent.com/Proxima-Engine/proxima-engine/main/install-customer.ps1) if it is not already present in the ZIP. The [customer Compose file](https://raw.githubusercontent.com/Proxima-Engine/proxima-engine/main/compose.customer.yaml) must remain beside it. -4. In the extracted folder, create a directory named `secrets`. Place the downloaded secret files there using the exact extensionless filenames: - - `secrets/node_enrollment_token` - - `secrets/tenant_signing_key` -5. Place the downloaded `.env` in the same folder as `install-customer.ps1` and `compose.customer.yaml`. Keep this folder private. In File Explorer, enable **File name extensions** so Windows does not silently append `.txt` to the secret filenames. +2. In Agata Proxima Cloud → Nodes, copy the Windows PowerShell setup commands shown with the node's one-time enrollment token. +3. Keep the included [Windows PowerShell installer](https://raw.githubusercontent.com/Proxima-Engine/proxima-engine/main/install-customer.ps1) and [customer Compose file](https://raw.githubusercontent.com/Proxima-Engine/proxima-engine/main/compose.customer.yaml) together in the extracted repository folder. +4. Paste the copied setup commands into the same PowerShell session. The installer generates `.env` and `secrets/node_enrollment_token`; it prompts for the PostgreSQL upstream address if it is not set. No tenant signing key download is required. ### Windows prerequisites - Windows 10/11 with Docker Desktop configured to use the WSL 2 backend and Linux containers. - Docker daemon running before installation. -- The node-specific `.env` and both secret files from Node Management. +- The Windows setup commands copied from the registered node in Node Management. - Network access from the Engine container to the configured PostgreSQL upstream and Agata Proxima HTTPS enrollment/heartbeat endpoints. The Engine is distributed as a Linux container. On Windows, Docker Desktop runs that container through its Linux/WSL 2 backend; this is the supported Windows installation path. The PowerShell installer is a Windows host installer, not an MSI and not a native Windows service binary. @@ -197,15 +194,16 @@ Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass The installer performs these checks and actions in order: -1. Confirms that `.env`, `compose.customer.yaml`, and both secret files exist. -2. Confirms the Docker CLI and a running Docker daemon. -3. Confirms that Docker is using Linux containers and Docker Compose v2 is available. -4. Validates the customer Compose configuration. -5. Builds and starts the Engine container. -6. Waits up to 120 seconds for the container's Docker health check to report `healthy`. -7. Prints container status and the local PostgreSQL proxy endpoint. +1. Generates the node environment file and one-time enrollment-token secret from the copied setup commands. +2. Prompts for the PostgreSQL upstream host and port when it is not already configured. +3. Confirms the Docker CLI and a running Docker daemon. +4. Confirms that Docker is using Linux containers and Docker Compose v2 is available. +5. Validates the customer Compose configuration. +6. Builds and starts the Engine container. +7. Waits up to 120 seconds for the container's Docker health check to report `healthy`. +8. Prints container status and the local PostgreSQL proxy endpoint. -A successful installation prints **Proxima Engine is healthy**. Then open Agata Proxima → Node Management/Fleet and confirm the registered node reports healthy after its signed heartbeat arrives. The one-time enrollment token is consumed during first enrollment; the persistent node identity is stored in a named Docker volume. Keep the tenant signing key mounted and protected for subsequent starts. +A successful installation prints **Proxima Engine is healthy**. Then open Agata Proxima → Nodes/Fleet and confirm the registered node reports healthy after its signed heartbeat arrives. During first enrollment, the Control Plane returns the node heartbeat key and an organization-scoped tenant verification key over HTTPS. The Engine stores both in its protected identity file in a named Docker volume. The Control Plane master signing secret is never sent to customer nodes. After the node is healthy, replace the contents of `secrets/node_enrollment_token` with `enrollment-complete`; keep the file in place for subsequent Compose starts. ### Verify and operate on Windows @@ -233,7 +231,7 @@ docker compose -f compose.customer.yaml up -d ### Linux installation -Linux customers use the matching Bash installer after placing the same four configuration/secret files in the extracted repository folder: +Linux customers copy the Linux shell setup commands from Agata Proxima → Nodes and paste them into a shell in the extracted repository folder. The Bash installer creates the environment and enrollment-token secret automatically: ```bash chmod +x ./install-customer.sh @@ -248,5 +246,5 @@ For the Linux deployment-executor service installer and its security requirement - If the application runs in another container, attach it to the configured Compose network and use the Engine service name and port. For a different host, use a private network/VPN, a deliberate private bind address, and a restrictive firewall rule. - The configured PostgreSQL upstream address is resolved from inside the Engine container. `localhost` refers to the container itself, not the Windows host. Use a reachable database hostname or the appropriate Docker Desktop host gateway address. - Never expose PostgreSQL proxy port 6432, the status interface, or metrics publicly. -- Never commit, email, or paste `.env`, enrollment tokens, tenant signing keys, node identity files, operator API keys, or webhook secrets. -- The tenant signing key must match the key used by the trusted application backend to sign tenant-context tokens. It is distinct from the one-time enrollment token and the node identity signing key. +- Never commit, email, or paste `.env`, enrollment tokens, node identity files, operator API keys, or webhook secrets. +- The Control Plane master tenant-signing secret stays in the Control Plane. A customer Engine receives only its own organization's derived verification key during HTTPS enrollment; the Engine stores it in the protected identity file. From 73ac63bba37effb8f0e93e931a9c6aeb4cd18299 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:53:03 -0800 Subject: [PATCH 089/108] Bootstrap node identity before loading Engine security configuration --- crates/proxima-engine/src/main.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/proxima-engine/src/main.rs b/crates/proxima-engine/src/main.rs index c201e6b..3b1b93e 100644 --- a/crates/proxima-engine/src/main.rs +++ b/crates/proxima-engine/src/main.rs @@ -352,6 +352,13 @@ async fn main() -> io::Result<()> { rustls::crypto::CryptoProvider::install_default(rustls::crypto::aws_lc_rs::default_provider()) .map_err(|_| io::Error::other("failed to install rustls AWS-LC crypto provider"))?; + // Enrollment must run before configuration is loaded: it supplies the + // node identity and organization-scoped tenant verification key. + if let Err(error) = bootstrap_node_identity().await { + eprintln!("Engine node bootstrap failed: {error}"); + std::process::exit(1); + } + let config = Config::from_env()?; let config_tenant_role_prefix = config.tenant_role_prefix.clone(); @@ -408,11 +415,6 @@ async fn main() -> io::Result<()> { .timeout(std::time::Duration::from_secs(5)) .build() .ok(); - if let Err(error) = bootstrap_node_identity().await { - eprintln!("Engine node bootstrap failed: {error}"); - std::process::exit(1); - } - if let (Some(url), Some(token), Some(client)) = ( security_event_url.clone(), security_event_token.clone(), From 3aa97545473d14af0cf6394d4c6f2e2321b84071 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:54:12 -0800 Subject: [PATCH 090/108] Add Linux installer smoke test for generated customer setup files --- tests/linux-installer-smoke.sh | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/linux-installer-smoke.sh diff --git a/tests/linux-installer-smoke.sh b/tests/linux-installer-smoke.sh new file mode 100644 index 0000000..3a24d6c --- /dev/null +++ b/tests/linux-installer-smoke.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +source_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +temp_root="$(mktemp -d)" +trap 'rm -rf "$temp_root"' EXIT +cp "$source_root/install-customer.sh" "$source_root/compose.customer.yaml" "$temp_root/" +cd "$temp_root" + +docker() { + case "$*" in + "compose version") echo "Docker Compose version v2.30.0" ;; + "compose -f compose.customer.yaml config --quiet") return 0 ;; + "compose -f compose.customer.yaml up -d --build") return 0 ;; + "compose -f compose.customer.yaml ps --format json engine") echo '{"Health":"healthy"}' ;; + "compose -f compose.customer.yaml ps") echo "engine running (healthy)" ;; + "compose -f compose.customer.yaml logs --tail 100 engine") echo "simulated engine logs" ;; + *) echo "Unexpected Docker invocation: $*" >&2; return 1 ;; + esac +} +export -f docker + +PROXIMA_NODE_ID=00000000-0000-4000-8000-000000000001 \ +PROXIMA_NODE_ENROLLMENT_URL=https://api.agataproxima.com/api/v1/nodes/enroll \ +PROXIMA_NODE_ENROLLMENT_TOKEN=test-one-time-token \ +PROXIMA_UPSTREAM_ADDR=postgres:5432 \ +bash ./install-customer.sh + +grep -Fxq "PROXIMA_NODE_ID=00000000-0000-4000-8000-000000000001" .env +grep -Fxq "PROXIMA_UPSTREAM_ADDR=postgres:5432" .env +[[ "$(cat secrets/node_enrollment_token)" == "test-one-time-token" ]] +[[ "$(stat -c '%a' secrets/node_enrollment_token)" == "600" ]] +[[ ! -e secrets/tenant_signing_key ]] + +echo "PASS: Linux customer installer creates the environment and owner-only enrollment secret, validates Compose, starts the Engine and checks health." From 2983c8ab96cac7b21b0fa6c200802b0d9bd33165 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:54:15 -0800 Subject: [PATCH 091/108] Run Linux installer generation and health-check smoke test in CI --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6f5714..eca3c78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,7 @@ jobs: run: | bash -n install-customer.sh bash -n install-deployment-executor.sh + bash tests/linux-installer-smoke.sh pwsh -NoProfile -Command '$tokens = $null; $errors = $null; [System.Management.Automation.Language.Parser]::ParseFile((Resolve-Path "install-customer.ps1"), [ref]$tokens, [ref]$errors) > $null; if ($errors.Count -gt 0) { $errors | ForEach-Object { Write-Error $_ }; exit 1 }' - name: Validate customer Compose installation contract run: | From 16c7828e2104ab9475c26c442d08e4bf085bb341 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:56:45 -0800 Subject: [PATCH 092/108] Refresh organization-scoped tenant key for existing node identities --- crates/proxima-engine/src/main.rs | 159 ++++++++++++++++++++++-------- 1 file changed, 118 insertions(+), 41 deletions(-) diff --git a/crates/proxima-engine/src/main.rs b/crates/proxima-engine/src/main.rs index 3b1b93e..6e2aefe 100644 --- a/crates/proxima-engine/src/main.rs +++ b/crates/proxima-engine/src/main.rs @@ -193,18 +193,45 @@ fn attestation_ingest_url_allowed(url: &str, allow_insecure_http: bool) -> bool } async fn bootstrap_node_identity() -> Result<(), Box> { - use std::{env, fs, path::PathBuf}; + use std::{ + env, fs, + io::Write, + path::{Path, PathBuf}, + time::{SystemTime, UNIX_EPOCH}, + }; - let identity_path = env::var_os("PROXIMA_NODE_IDENTITY_PATH") - .map(PathBuf::from) - .or_else(|| { - env::var_os("LOCALAPPDATA").map(|base| { - PathBuf::from(base) - .join("AgataProxima") - .join("node-identity.json") - }) - }) - .unwrap_or_else(|| PathBuf::from("data/node-identity.json")); + fn persist_identity( + identity_path: &Path, + identity: &serde_json::Value, + ) -> Result<(), Box> { + let parent = identity_path.parent().ok_or("Invalid node identity path")?; + fs::create_dir_all(parent)?; + let temporary_path = identity_path.with_extension("json.tmp"); + if temporary_path.exists() { + fs::remove_file(&temporary_path)?; + } + let identity_bytes = serde_json::to_vec_pretty(identity)?; + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary_path)?; + file.write_all(&identity_bytes)?; + file.sync_all()?; + } + #[cfg(not(unix))] + fs::write(&temporary_path, &identity_bytes)?; + + #[cfg(windows)] + if identity_path.exists() { + fs::remove_file(identity_path)?; + } + fs::rename(&temporary_path, identity_path)?; + Ok(()) + } fn apply_identity( value: &serde_json::Value, @@ -230,18 +257,91 @@ async fn bootstrap_node_identity() -> Result<(), Box::new_from_slice(&node_key) + .map_err(|_| "Invalid node signing key")?; + mac.update(canonical.as_bytes()); + let signature = hex::encode(mac.finalize().into_bytes()); + + let response = reqwest::Client::new() + .post(heartbeat_url) + .json(&serde_json::json!({ + "node_id": node_id, + "timestamp": timestamp, + "version": version, + "runtime_status": runtime_status, + "signature": signature, + "include_tenant_signing_key": true + })) + .send() + .await? + .error_for_status()?; + let refresh: serde_json::Value = response.json().await?; + let tenant_key = refresh["tenant_signing_key"] + .as_str() + .ok_or("Control Plane identity refresh response is missing tenant_signing_key")?; + if hex::decode(tenant_key)?.len() != 32 { + return Err("Control Plane returned an invalid organization tenant signing key".into()); + } + identity["tenant_signing_key"] = serde_json::Value::String(tenant_key.to_owned()); + persist_identity(&identity_path, &identity)?; + } + apply_identity(&identity)?; #[cfg(unix)] { @@ -289,7 +389,7 @@ async fn bootstrap_node_identity() -> Result<(), Box Result<(), Box Date: Tue, 10 Nov 2026 22:57:17 -0800 Subject: [PATCH 093/108] Document automatic identity-key refresh for existing Engine nodes --- docs/NODE_ENROLLMENT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/NODE_ENROLLMENT.md b/docs/NODE_ENROLLMENT.md index 0114f84..8f014c4 100644 --- a/docs/NODE_ENROLLMENT.md +++ b/docs/NODE_ENROLLMENT.md @@ -107,7 +107,7 @@ Default identity paths: Set `PROXIMA_NODE_IDENTITY_PATH` to choose another path. The identity file contains the node ID, organization ID, the 32-byte node heartbeat key and the 32-byte organization-scoped tenant-token verification key. Restrict file permissions and include the file in protected backups. The Control Plane master secret is not stored on the Engine. -The Engine reuses a valid saved identity on later starts, so the one-time enrollment token is needed only for the initial bootstrap. Use a separate identity path for each node. +The Engine reuses a valid saved identity on later starts, so the one-time enrollment token is needed only for the initial bootstrap. If an existing identity predates organization-scoped tenant keys, the updated Engine uses a signed, domain-separated heartbeat refresh to obtain the key for its organization and updates the identity file without replacing the node UUID or consuming another enrollment token. Use a separate identity path for each node. When the Engine has a registered node identity and a compatible heartbeat endpoint, it sends an HMAC-SHA256 signed liveness report every 30 seconds. The heartbeat uses the enrolled node signing key, is independent of the hourly attestation ledger, and does not assert tenant isolation. Fleet marks a node degraded after 90 seconds without a heartbeat and offline after five minutes. The control plane rejects stale, replayed, incorrectly signed, or revoked-node heartbeats. From cefa843a6fe26f564941563f1153e3768d344358 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 22:57:58 -0800 Subject: [PATCH 094/108] Scope secure identity file write imports to Unix builds --- crates/proxima-engine/src/main.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/proxima-engine/src/main.rs b/crates/proxima-engine/src/main.rs index 6e2aefe..626bde4 100644 --- a/crates/proxima-engine/src/main.rs +++ b/crates/proxima-engine/src/main.rs @@ -195,7 +195,6 @@ fn attestation_ingest_url_allowed(url: &str, allow_insecure_http: bool) -> bool async fn bootstrap_node_identity() -> Result<(), Box> { use std::{ env, fs, - io::Write, path::{Path, PathBuf}, time::{SystemTime, UNIX_EPOCH}, }; @@ -213,7 +212,7 @@ async fn bootstrap_node_identity() -> Result<(), Box Date: Tue, 10 Nov 2026 23:01:13 -0800 Subject: [PATCH 095/108] Add Linux daemon checks and immediate container failure diagnostics --- install-customer.sh | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/install-customer.sh b/install-customer.sh index d45f34f..6239e3a 100644 --- a/install-customer.sh +++ b/install-customer.sh @@ -31,17 +31,33 @@ fi unset PROXIMA_NODE_ENROLLMENT_TOKEN || true docker compose version >/dev/null +docker_os="$(docker info --format '{{.OSType}}')" || { + echo "Docker daemon is unavailable. Start Docker Engine or Docker Desktop and rerun this installer." >&2 + exit 1 +} +[[ "$docker_os" == "linux" ]] || { + echo "Proxima Engine requires Linux containers." >&2 + exit 1 +} docker compose -f compose.customer.yaml config --quiet echo "Building and starting Proxima Engine. The first build downloads Rust dependencies and can take several minutes." -docker compose -f compose.customer.yaml up -d --build +if ! docker compose -f compose.customer.yaml up -d --build; then + docker compose -f compose.customer.yaml logs --tail 100 engine || true + echo "Engine container failed to start. Review the logs above and correct the reported issue before retrying." >&2 + exit 1 +fi echo "Waiting for Engine health check..." for attempt in $(seq 1 40); do - if docker compose -f compose.customer.yaml ps --format json engine | grep -q '"Health":"healthy"'; then + container_state="$(docker compose -f compose.customer.yaml ps --format json engine 2>/dev/null || true)" + if grep -q '"Health":"healthy"' <<<"$container_state"; then docker compose -f compose.customer.yaml ps echo "Proxima Engine is healthy. Node enrollment and signed heartbeat are active. The organization-scoped tenant verification key is stored in the protected node identity file." exit 0 fi + if grep -q '"Health":"unhealthy"' <<<"$container_state"; then + break + fi sleep 3 done docker compose -f compose.customer.yaml ps From 9aaca249d39bcc88cf764a76589af3d40d0a36db Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 23:01:21 -0800 Subject: [PATCH 096/108] Mock Linux Docker daemon information in installer smoke test --- tests/linux-installer-smoke.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/linux-installer-smoke.sh b/tests/linux-installer-smoke.sh index 3a24d6c..0c4bb7a 100644 --- a/tests/linux-installer-smoke.sh +++ b/tests/linux-installer-smoke.sh @@ -9,6 +9,7 @@ cd "$temp_root" docker() { case "$*" in + "info --format {{.OSType}}") echo "linux" ;; "compose version") echo "Docker Compose version v2.30.0" ;; "compose -f compose.customer.yaml config --quiet") return 0 ;; "compose -f compose.customer.yaml up -d --build") return 0 ;; From 93f83e2cb19bd1ed374fd689f8867a3f413163bf Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 23:01:51 -0800 Subject: [PATCH 097/108] Persist the explicit node heartbeat URL in customer environment --- install-customer.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/install-customer.ps1 b/install-customer.ps1 index 44165ea..bd69126 100644 --- a/install-customer.ps1 +++ b/install-customer.ps1 @@ -19,6 +19,9 @@ if (-not (Test-Path -LiteralPath (Join-Path $root ".env") -PathType Leaf)) { if ([string]::IsNullOrWhiteSpace($env:PROXIMA_NODE_ENROLLMENT_URL)) { $env:PROXIMA_NODE_ENROLLMENT_URL = "https://api.agataproxima.com/api/v1/nodes/enroll" } + if ([string]::IsNullOrWhiteSpace($env:PROXIMA_NODE_HEARTBEAT_URL)) { + $env:PROXIMA_NODE_HEARTBEAT_URL = $env:PROXIMA_NODE_ENROLLMENT_URL.Replace("/api/v1/nodes/enroll", "/api/v1/nodes/heartbeat") + } if ([string]::IsNullOrWhiteSpace($env:PROXIMA_UPSTREAM_ADDR)) { $env:PROXIMA_UPSTREAM_ADDR = Read-Host "PostgreSQL upstream host:port (for example, db.example.com:5432)" } @@ -31,6 +34,7 @@ if (-not (Test-Path -LiteralPath (Join-Path $root ".env") -PathType Leaf)) { @( "PROXIMA_NODE_ID=$($env:PROXIMA_NODE_ID)" "PROXIMA_NODE_ENROLLMENT_URL=$($env:PROXIMA_NODE_ENROLLMENT_URL)" + "PROXIMA_NODE_HEARTBEAT_URL=$($env:PROXIMA_NODE_HEARTBEAT_URL)" "PROXIMA_UPSTREAM_ADDR=$($env:PROXIMA_UPSTREAM_ADDR)" "PROXIMA_LISTEN_ADDR=$($env:PROXIMA_LISTEN_ADDR)" "PROXIMA_ENVIRONMENT=production" From 68198652cc7918a3d8c7164316d8e8fabecd6dd5 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 23:01:53 -0800 Subject: [PATCH 098/108] Persist the explicit node heartbeat URL in customer environment --- install-customer.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/install-customer.sh b/install-customer.sh index 6239e3a..868d754 100644 --- a/install-customer.sh +++ b/install-customer.sh @@ -8,6 +8,7 @@ command -v docker >/dev/null 2>&1 || { echo "Docker CLI is required." >&2; exit if [[ ! -f .env ]]; then : "${PROXIMA_NODE_ID:?Set PROXIMA_NODE_ID using the enrollment configuration copied from Agata Proxima → Nodes}" PROXIMA_NODE_ENROLLMENT_URL="${PROXIMA_NODE_ENROLLMENT_URL:-https://api.agataproxima.com/api/v1/nodes/enroll}" + PROXIMA_NODE_HEARTBEAT_URL="${PROXIMA_NODE_HEARTBEAT_URL:-${PROXIMA_NODE_ENROLLMENT_URL%/enroll}/heartbeat}" if [[ -z "${PROXIMA_UPSTREAM_ADDR:-}" ]]; then read -r -p "PostgreSQL upstream host:port (for example, db.example.com:5432): " PROXIMA_UPSTREAM_ADDR fi @@ -15,6 +16,7 @@ if [[ ! -f .env ]]; then cat > .env < Date: Tue, 10 Nov 2026 23:01:55 -0800 Subject: [PATCH 099/108] Verify Windows installer persists the heartbeat endpoint --- tests/windows-installer-smoke.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/windows-installer-smoke.ps1 b/tests/windows-installer-smoke.ps1 index 045d3d1..e7e4ad9 100644 --- a/tests/windows-installer-smoke.ps1 +++ b/tests/windows-installer-smoke.ps1 @@ -35,6 +35,9 @@ try { if ($generatedEnv -notmatch "PROXIMA_UPSTREAM_ADDR=postgres:5432") { throw "Windows installer did not generate the PostgreSQL upstream setting." } + if ($generatedEnv -notmatch "PROXIMA_NODE_HEARTBEAT_URL=https://api.agataproxima.com/api/v1/nodes/heartbeat") { + throw "Windows installer did not persist the node heartbeat URL." + } if ($generatedToken.Trim() -ne "test-one-time-token") { throw "Windows installer did not create the one-time enrollment secret file correctly." } From 3b052782b694011f4486803822fcf6eec3803994 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 23:01:58 -0800 Subject: [PATCH 100/108] Verify Linux installer persists the heartbeat endpoint --- tests/linux-installer-smoke.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/linux-installer-smoke.sh b/tests/linux-installer-smoke.sh index 0c4bb7a..5506fbc 100644 --- a/tests/linux-installer-smoke.sh +++ b/tests/linux-installer-smoke.sh @@ -29,6 +29,7 @@ bash ./install-customer.sh grep -Fxq "PROXIMA_NODE_ID=00000000-0000-4000-8000-000000000001" .env grep -Fxq "PROXIMA_UPSTREAM_ADDR=postgres:5432" .env +grep -Fxq "PROXIMA_NODE_HEARTBEAT_URL=https://api.agataproxima.com/api/v1/nodes/heartbeat" .env [[ "$(cat secrets/node_enrollment_token)" == "test-one-time-token" ]] [[ "$(stat -c '%a' secrets/node_enrollment_token)" == "600" ]] [[ ! -e secrets/tenant_signing_key ]] From ef3f3379de7f3d69de96637f843390141d85a1b5 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 23:02:05 -0800 Subject: [PATCH 101/108] Point issue reporting to the Proxima Engine repository --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dc49959..73a311d 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ The Proxima Engine source and covered documentation are distributed under the [M ## Contribute -Use [GitHub Issues](https://github.com/Agata-Proxima/proxima-engine/issues) for reproducible Engine bugs and feature requests. Read [Contributing](CONTRIBUTING.md) before submitting a pull request. For vulnerabilities, use the private reporting process in [SECURITY.md](SECURITY.md). +Use [GitHub Issues](https://github.com/Proxima-Engine/proxima-engine/issues) for reproducible Engine bugs and feature requests. Read [Contributing](CONTRIBUTING.md) before submitting a pull request. For vulnerabilities, use the private reporting process in [SECURITY.md](SECURITY.md).

Built by Agata Proxima · agataproxima.com · GitHub organization

From a1a533a2bc24e44a34ba190949f07a7aa7fc3f42 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 23:42:18 -0800 Subject: [PATCH 102/108] feat(engine): accept legacy tenant tokens during key migration --- crates/proxima-engine/src/tenant.rs | 33 +++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/crates/proxima-engine/src/tenant.rs b/crates/proxima-engine/src/tenant.rs index d5a8a40..0d8c1b1 100644 --- a/crates/proxima-engine/src/tenant.rs +++ b/crates/proxima-engine/src/tenant.rs @@ -58,6 +58,7 @@ impl std::error::Error for TenantTokenError {} #[derive(Clone)] pub struct TenantTokenVerifier { secret: Vec, + previous_secret: Option>, replay_dir: PathBuf, } @@ -73,6 +74,25 @@ impl TenantTokenVerifier { .unwrap_or_else(|| PathBuf::from("data/proxima-break-glass-replays")); Ok(Self { secret: secret.to_vec(), + previous_secret: None, + replay_dir, + }) + } + + pub fn new_with_previous( + secret: impl AsRef<[u8]>, + previous_secret: Option<&[u8]>, + ) -> Result { + let secret = secret.as_ref(); + if secret.len() < 32 || previous_secret.is_some_and(|previous| previous.len() < 32) { + return Err(TenantTokenError::InvalidSignature); + } + let replay_dir = std::env::var_os("PROXIMA_BREAK_GLASS_REPLAY_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from("data/proxima-break-glass-replays")); + Ok(Self { + secret: secret.to_vec(), + previous_secret: previous_secret.map(ToOwned::to_owned), replay_dir, }) } @@ -87,6 +107,7 @@ impl TenantTokenVerifier { } Ok(Self { secret: secret.to_vec(), + previous_secret: None, replay_dir, }) } @@ -124,7 +145,11 @@ impl TenantTokenVerifier { } let expected = sign_payload(&self.secret, version, tenant_id, expires_at); - if !constant_time_hex_eq(&expected, signature) { + let valid_current = constant_time_hex_eq(&expected, signature); + let valid_previous = self.previous_secret.as_ref().is_some_and(|secret| { + constant_time_hex_eq(&sign_payload(secret, version, tenant_id, expires_at), signature) + }); + if !valid_current && !valid_previous { return Err(TenantTokenError::InvalidSignature); } @@ -166,7 +191,11 @@ impl TenantTokenVerifier { return Err(TenantTokenError::InvalidFormat); } let payload = format!("bg1.{tenant_id}.{expires_at}.{nonce}"); - if !constant_time_hex_eq(&sign_message(&self.secret, &payload), parts[4]) { + let valid_current = constant_time_hex_eq(&sign_message(&self.secret, &payload), parts[4]); + let valid_previous = self.previous_secret.as_ref().is_some_and(|secret| { + constant_time_hex_eq(&sign_message(secret, &payload), parts[4]) + }); + if !valid_current && !valid_previous { return Err(TenantTokenError::InvalidSignature); } fs::create_dir_all(&self.replay_dir) From 77ce4cd3a8d3364fd9ed83d5ae73855a54252357 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 23:42:29 -0800 Subject: [PATCH 103/108] feat(engine): configure previous tenant signing key for safe rotation --- crates/proxima-engine/src/config.rs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/crates/proxima-engine/src/config.rs b/crates/proxima-engine/src/config.rs index 7596863..730b4c5 100644 --- a/crates/proxima-engine/src/config.rs +++ b/crates/proxima-engine/src/config.rs @@ -29,6 +29,7 @@ pub struct Config { pub listen_addr: SocketAddr, pub upstream_addr: String, pub tenant_signing_key: Option, + pub tenant_signing_key_previous: Option, pub tenant_role_prefix: String, pub upstream_connect_timeout: Duration, pub max_connections: usize, @@ -78,9 +79,14 @@ impl Config { )); } - let tenant_signing_key_value = match env::var("PROXIMA_TENANT_SIGNING_KEY_FILE") { - Ok(path) => Some(std::fs::read_to_string(path)?.trim().to_owned()), - Err(_) => env::var("PROXIMA_TENANT_SIGNING_KEY").ok(), + // A node identity loaded during bootstrap sets the organization key in + // the environment. It must take precedence over any legacy key file. + let tenant_signing_key_value = match env::var("PROXIMA_TENANT_SIGNING_KEY") { + Ok(value) => Some(value), + Err(_) => match env::var("PROXIMA_TENANT_SIGNING_KEY_FILE") { + Ok(path) => Some(std::fs::read_to_string(path)?.trim().to_owned()), + Err(_) => None, + }, }; let tenant_signing_key = match tenant_signing_key_value { Some(value) if value.trim().is_empty() => None, @@ -93,6 +99,13 @@ impl Config { Some(value) => Some(value), None => None, }; + let tenant_signing_key_previous = non_empty_env("PROXIMA_TENANT_SIGNING_KEY_PREVIOUS"); + if tenant_signing_key_previous.as_ref().is_some_and(|key| key.len() < 32) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "PROXIMA_TENANT_SIGNING_KEY_PREVIOUS must contain at least 32 bytes", + )); + } let upstream_connect_timeout_ms = env::var("PROXIMA_UPSTREAM_CONNECT_TIMEOUT_MS") .unwrap_or_else(|_| "10000".to_string()) @@ -179,6 +192,7 @@ impl Config { listen_addr, upstream_addr, tenant_signing_key, + tenant_signing_key_previous, tenant_role_prefix, upstream_connect_timeout: Duration::from_millis(upstream_connect_timeout_ms), max_connections, @@ -246,6 +260,7 @@ mod tests { "PROXIMA_UPSTREAM_ADDR", "PROXIMA_TENANT_SIGNING_KEY", "PROXIMA_TENANT_SIGNING_KEY_FILE", + "PROXIMA_TENANT_SIGNING_KEY_PREVIOUS", "PROXIMA_UPSTREAM_CONNECT_TIMEOUT_MS", "PROXIMA_MAX_CONNECTIONS", "PROXIMA_TLS_CERT_FILE", @@ -266,6 +281,7 @@ mod tests { assert_eq!(config.listen_addr, "127.0.0.1:6432".parse().unwrap()); assert_eq!(config.upstream_addr, "127.0.0.1:5432"); assert_eq!(config.tenant_signing_key, None); + assert_eq!(config.tenant_signing_key_previous, None); assert_eq!(config.tenant_role_prefix, "proxima_tenant_"); assert_eq!(config.upstream_connect_timeout, Duration::from_secs(10)); assert_eq!(config.max_connections, 1024); From ce7b6340b95e85f38de0e12e309364e2fcf66fd6 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 23:42:41 -0800 Subject: [PATCH 104/108] feat(engine): preserve legacy token verification during node identity refresh --- crates/proxima-engine/src/main.rs | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/crates/proxima-engine/src/main.rs b/crates/proxima-engine/src/main.rs index 626bde4..aa2d73f 100644 --- a/crates/proxima-engine/src/main.rs +++ b/crates/proxima-engine/src/main.rs @@ -258,6 +258,12 @@ async fn bootstrap_node_identity() -> Result<(), Box Result<(), Box= 32) { + identity["legacy_tenant_signing_key"] = serde_json::Value::String(key); + } + } // Upgrade an identity created before organization-scoped tenant keys // were returned during enrollment. This signed, domain-separated // refresh does not require consuming another enrollment token. @@ -441,7 +460,12 @@ async fn main() -> io::Result<()> { let verifier = config .tenant_signing_key .as_deref() - .map(TenantTokenVerifier::new) + .map(|key| { + TenantTokenVerifier::new_with_previous( + key, + config.tenant_signing_key_previous.as_deref(), + ) + }) .transpose() .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error.to_string()))?; From 9ddb7bcf442ec88226108827e0328a9f82be11a8 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 23:42:49 -0800 Subject: [PATCH 105/108] test(engine): cover legacy token verification during key rotation --- crates/proxima-engine/src/tenant.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/proxima-engine/src/tenant.rs b/crates/proxima-engine/src/tenant.rs index 0d8c1b1..ce87d64 100644 --- a/crates/proxima-engine/src/tenant.rs +++ b/crates/proxima-engine/src/tenant.rs @@ -400,3 +400,26 @@ mod tests { ); } } + +#[cfg(test)] +mod previous_key_tests { + use super::*; + + #[test] + fn accepts_existing_tokens_with_previous_key_during_rotation() { + let previous = b"legacy-global-tenant-signing-key-0001"; + let current = b"organization-scoped-tenant-signing-key-01"; + let verifier = TenantTokenVerifier::new_with_previous(current, Some(previous)).unwrap(); + let token = format!("v1.tenant_a.2000.{}", sign_payload(previous, "v1", "tenant_a", 2000)); + assert_eq!(verifier.verify(&token, 1000).unwrap().tenant_id, "tenant_a"); + } + + #[test] + fn rejects_previous_key_when_not_configured() { + let previous = b"legacy-global-tenant-signing-key-0001"; + let current = b"organization-scoped-tenant-signing-key-01"; + let verifier = TenantTokenVerifier::new(current).unwrap(); + let token = format!("v1.tenant_a.2000.{}", sign_payload(previous, "v1", "tenant_a", 2000)); + assert_eq!(verifier.verify(&token, 1000), Err(TenantTokenError::InvalidSignature)); + } +} From 118d32308527dc4d7bf51f248e92312c1e0b162a Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 23:43:58 -0800 Subject: [PATCH 106/108] fix(engine): accept previous signing key as string configuration --- crates/proxima-engine/src/tenant.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/proxima-engine/src/tenant.rs b/crates/proxima-engine/src/tenant.rs index ce87d64..5048ba5 100644 --- a/crates/proxima-engine/src/tenant.rs +++ b/crates/proxima-engine/src/tenant.rs @@ -81,10 +81,10 @@ impl TenantTokenVerifier { pub fn new_with_previous( secret: impl AsRef<[u8]>, - previous_secret: Option<&[u8]>, + previous_secret: Option<&str>, ) -> Result { let secret = secret.as_ref(); - if secret.len() < 32 || previous_secret.is_some_and(|previous| previous.len() < 32) { + if secret.len() < 32 || previous_secret.is_some_and(|previous| previous.as_bytes().len() < 32) { return Err(TenantTokenError::InvalidSignature); } let replay_dir = std::env::var_os("PROXIMA_BREAK_GLASS_REPLAY_DIR") @@ -92,7 +92,7 @@ impl TenantTokenVerifier { .unwrap_or_else(|| PathBuf::from("data/proxima-break-glass-replays")); Ok(Self { secret: secret.to_vec(), - previous_secret: previous_secret.map(ToOwned::to_owned), + previous_secret: previous_secret.map(|previous| previous.as_bytes().to_vec()), replay_dir, }) } From 278081fa88e4372ba4f8af542a7d804a8dbaa7a9 Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 23:44:01 -0800 Subject: [PATCH 107/108] test(engine): update TLS config fixture for key rotation --- crates/proxima-engine/src/tls.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/proxima-engine/src/tls.rs b/crates/proxima-engine/src/tls.rs index 511d2d9..c90878d 100644 --- a/crates/proxima-engine/src/tls.rs +++ b/crates/proxima-engine/src/tls.rs @@ -166,6 +166,7 @@ mod tests { listen_addr: "127.0.0.1:6432".parse().unwrap(), upstream_addr: "127.0.0.1:5432".into(), tenant_signing_key: None, + tenant_signing_key_previous: None, tenant_role_prefix: "proxima_tenant_".into(), upstream_connect_timeout: std::time::Duration::from_secs(1), max_connections: 10, From cab8667b51a3759e12e955c975480c5323408daa Mon Sep 17 00:00:00 2001 From: Anthony Emmanuella Mmasinachi Date: Tue, 10 Nov 2026 23:44:09 -0800 Subject: [PATCH 108/108] test(engine): align previous-key tests with string config --- crates/proxima-engine/src/tenant.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/proxima-engine/src/tenant.rs b/crates/proxima-engine/src/tenant.rs index 5048ba5..ea6dfd6 100644 --- a/crates/proxima-engine/src/tenant.rs +++ b/crates/proxima-engine/src/tenant.rs @@ -407,19 +407,19 @@ mod previous_key_tests { #[test] fn accepts_existing_tokens_with_previous_key_during_rotation() { - let previous = b"legacy-global-tenant-signing-key-0001"; - let current = b"organization-scoped-tenant-signing-key-01"; + let previous = "legacy-global-tenant-signing-key-0001"; + let current = "organization-scoped-tenant-signing-key-01"; let verifier = TenantTokenVerifier::new_with_previous(current, Some(previous)).unwrap(); - let token = format!("v1.tenant_a.2000.{}", sign_payload(previous, "v1", "tenant_a", 2000)); + let token = format!("v1.tenant_a.2000.{}", sign_payload(previous.as_bytes(), "v1", "tenant_a", 2000)); assert_eq!(verifier.verify(&token, 1000).unwrap().tenant_id, "tenant_a"); } #[test] fn rejects_previous_key_when_not_configured() { - let previous = b"legacy-global-tenant-signing-key-0001"; - let current = b"organization-scoped-tenant-signing-key-01"; + let previous = "legacy-global-tenant-signing-key-0001"; + let current = "organization-scoped-tenant-signing-key-01"; let verifier = TenantTokenVerifier::new(current).unwrap(); - let token = format!("v1.tenant_a.2000.{}", sign_payload(previous, "v1", "tenant_a", 2000)); + let token = format!("v1.tenant_a.2000.{}", sign_payload(previous.as_bytes(), "v1", "tenant_a", 2000)); assert_eq!(verifier.verify(&token, 1000), Err(TenantTokenError::InvalidSignature)); } }