diff --git a/CHANGELOG.md b/CHANGELOG.md index abc3d2b8..2b6106f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,35 @@ All notable changes to this project will be documented in this file. ## [Unreleased] -### Added — Developer project synchronization +## [0.3.3] — 2026-09-18 + +### Added — Chat session management + +- Added chat session endpoints with archive and per-message encryption. +- Added Casbin RBAC rules for `/api/chat` routes. + +### Added — Agent hardening & ownership + +- Enforced per-tenant ownership on `/v1/agent` routes. +- Agent tokens now verified against a stored digest instead of round-tripping Vault. +- Agent authentication fails closed on Vault errors. +- Agent registration requires the service key; marketplace registration returns 501. +- Accept aggregate `all_health` report from agents for container status. +- Return `project_id` in agent snapshots and one-click clone endpoint. +- Container scope classified from Docker label, not container name. +- Moved `rotate-token` from the console binary to the CLI (`stacker agent rotate-token`). + +### Added — Marketplace field policy & secret federation + +- Added `config_contract` field-policy support (`fixed`/`editable`/`generated` + types: `hex`, `alphanumeric`, `uuid`, `derived_jwt`). +- At publish, `generated`-field values are stripped from the stored `stack_definition` (fail-closed). +- `derived_jwt` fields signed on cloned boxes via HMAC. +- Config contract federated to the User Service. +- Added `backfill_field_policy` one-shot tool to re-gate the existing catalog. +- Policy-driven `generate-secrets.sh` reads `mutability:generated` field policy instead of hardcoding `openssl rand`. +- Added `DisplayType` enum and `display` field to `FieldPolicy`. + +### Added — Project synchronization & one-click deploy - Added `stacker sync` to synchronize declarative project and app configuration with Stacker without creating a deployment, contacting a target server, or @@ -14,6 +42,57 @@ All notable changes to this project will be documented in this file. synchronization. - Added centralized sensitive environment-name redaction and validation for marketplace asset and seed-job metadata. +- One-click clone now registers the server in inventory and creates a cloud firewall. +- One-click clone seeds `project_app` records for the Applications panel. + +### Added — Deployment lifecycle & cleanup + +- Added `deployment_container` table to track containers per deployment (replaces name inference). +- Added deployment container sweeper for stale containers. +- Added stale project and server cleanup with notification (`cleanup-notify` binary). +- Added scheduled audit-log cleanup cron job. +- Added env size validator. + +### Added — Security & infrastructure + +- mTLS for Vault access; Vault client reports missing CA. +- `yaml_quote` now escapes control characters (`\n`, `\r`, `\t`). +- Docker preflight check (`docker info`) before deploy. +- New `W003` warning: `deploy.server.ssh_key` silently ignored on cloud deploys. + +### Fixed — SSH key authorization + +- Cloud deploy now fails when SSH key cannot be stored (was a silent warning). +- Cloud deploy fails when no SSH access is verified after provisioning. +- SSH key authorization retried while the VM boots; both Vault-managed and user keys authorized independently. +- User's configured SSH key from `deploy.cloud.ssh_key` authorized through the correct endpoint. + +### Fixed — CLI & config + +- Fixed `server --dry-run` no longer runs a real Docker deploy (#238). +- Fixed 500 on `PUT /cloud/{id}`: owner set before conversion. +- Fixed #251: escape env/label values so multiline config survives YAML. +- Ports validated by range, not by digit count. +- Port values in `stacker.yml` handled correctly when unquoted. +- Healthcheck `test` field emitted in the form docker compose expects (CMD list vs CMD-SHELL). + +### Fixed — Database & migrations + +- Fixed migration version collisions breaking CI. +- Guard optional cron job lookup to prevent panics. +- Reconcile audit-log cleanup cron after extension install. +- Restored `sqlx prepare` with missing `config_contract` field. + +### Fixed — Auth & Casbin + +- Added missing Casbin rules for admin `detect-secrets` endpoint. +- Agent auth accepts both Vault response shapes for the token. +- Credentials tests no longer read the developer's own config. + +### Fixed — Marketplace BDD + +- BDD marketplace analytics fixtures: cast `template_id` to UUID. +- BDD marketplace scenarios: seed `source_project_id` + deployment. ## [0.3.2] — 2026-08-26 diff --git a/Cargo.lock b/Cargo.lock index c54aea07..57cccdc4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7013,7 +7013,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.3.2" +version = "0.3.3" dependencies = [ "actix", "actix-casbin-auth", diff --git a/Cargo.toml b/Cargo.toml index c110271d..7f5fb148 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "stacker" -version = "0.3.2" +version = "0.3.3" edition = "2021" default-run= "server" diff --git a/migrations/20260917120000_fix_cleanup_type_mismatch_cleanup_log_retention.down.sql b/migrations/20260917120000_fix_cleanup_type_mismatch_cleanup_log_retention.down.sql new file mode 100644 index 00000000..24d0ff43 --- /dev/null +++ b/migrations/20260917120000_fix_cleanup_type_mismatch_cleanup_log_retention.down.sql @@ -0,0 +1,78 @@ +-- Revert: restore original stacker_cleanup_terminal_commands, drop cleanup_log cleanup + +-- Unschedule cleanup_log cron job +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_cron') THEN + PERFORM cron.unschedule('stacker_cleanup_cleanup_log'); + END IF; +EXCEPTION WHEN OTHERS THEN NULL; +END; +$$; + +-- Drop cleanup_log cleanup function +DROP FUNCTION IF EXISTS stacker_cleanup_cleanup_log(INTERVAL, BOOLEAN); + +-- Restore original stacker_cleanup_terminal_commands (with the type mismatch bug) +CREATE OR REPLACE FUNCTION stacker_cleanup_terminal_commands( + retention INTERVAL DEFAULT INTERVAL '30 days', + p_dry_run BOOLEAN DEFAULT true +) +RETURNS void LANGUAGE plpgsql AS $$ +DECLARE + v_count BIGINT; +BEGIN + IF p_dry_run THEN + SELECT count(*) INTO v_count FROM commands + WHERE status IN ('completed','failed','cancelled') + AND COALESCE(completed_at, updated_at) < NOW() - retention; + INSERT INTO cleanup_log (function_name, table_name, rows_deleted, retention, dry_run) + VALUES ('stacker_cleanup_terminal_commands', 'commands', v_count, retention, true); + + SELECT count(*) INTO v_count FROM dead_letter_queue + WHERE status IN ('exhausted','discarded','resolved') + AND updated_at < NOW() - retention; + INSERT INTO cleanup_log (function_name, table_name, rows_deleted, retention, dry_run) + VALUES ('stacker_cleanup_terminal_commands', 'dead_letter_queue', v_count, retention, true); + + SELECT count(*) INTO v_count FROM pipe_executions + WHERE completed_at < NOW() - retention; + INSERT INTO cleanup_log (function_name, table_name, rows_deleted, retention, dry_run) + VALUES ('stacker_cleanup_terminal_commands', 'pipe_executions', v_count, retention, true); + + SELECT count(*) INTO v_count FROM pipe_dag_step_executions + WHERE completed_at < NOW() - retention; + INSERT INTO cleanup_log (function_name, table_name, rows_deleted, retention, dry_run) + VALUES ('stacker_cleanup_terminal_commands', 'pipe_dag_step_executions', v_count, retention, true); + ELSE + -- Delete command_queue entries for terminal commands first (FK) + DELETE FROM command_queue WHERE command_id IN ( + SELECT id FROM commands + WHERE status IN ('completed','failed','cancelled') + AND COALESCE(completed_at, updated_at) < NOW() - retention + ); + + DELETE FROM commands + WHERE status IN ('completed','failed','cancelled') + AND COALESCE(completed_at, updated_at) < NOW() - retention; + GET DIAGNOSTICS v_count = ROW_COUNT; + INSERT INTO cleanup_log (function_name, table_name, rows_deleted, retention, dry_run) + VALUES ('stacker_cleanup_terminal_commands', 'commands', v_count, retention, false); + + DELETE FROM dead_letter_queue + WHERE status IN ('exhausted','discarded','resolved') + AND updated_at < NOW() - retention; + GET DIAGNOSTICS v_count = ROW_COUNT; + INSERT INTO cleanup_log (function_name, table_name, rows_deleted, retention, dry_run) + VALUES ('stacker_cleanup_terminal_commands', 'dead_letter_queue', v_count, retention, false); + + -- FK-safe order: step executions before executions + DELETE FROM pipe_dag_step_executions WHERE completed_at < NOW() - retention; + + DELETE FROM pipe_executions WHERE completed_at < NOW() - retention; + GET DIAGNOSTICS v_count = ROW_COUNT; + INSERT INTO cleanup_log (function_name, table_name, rows_deleted, retention, dry_run) + VALUES ('stacker_cleanup_terminal_commands', 'pipe_executions', v_count, retention, false); + END IF; +END; +$$; diff --git a/migrations/20260917120000_fix_cleanup_type_mismatch_cleanup_log_retention.up.sql b/migrations/20260917120000_fix_cleanup_type_mismatch_cleanup_log_retention.up.sql new file mode 100644 index 00000000..7a02df8d --- /dev/null +++ b/migrations/20260917120000_fix_cleanup_type_mismatch_cleanup_log_retention.up.sql @@ -0,0 +1,106 @@ +-- Fix stacker_cleanup_terminal_commands type mismatch and add cleanup_log retention + +-- 1. Fix stacker_cleanup_terminal_commands: command_queue.command_id is VARCHAR +-- referencing commands.command_id (VARCHAR), NOT commands.id (UUID). +-- The original function used `SELECT id FROM commands` which caused: +-- ERROR: operator does not exist: character varying = uuid +CREATE OR REPLACE FUNCTION stacker_cleanup_terminal_commands( + retention INTERVAL DEFAULT INTERVAL '30 days', + p_dry_run BOOLEAN DEFAULT true +) +RETURNS void LANGUAGE plpgsql AS $$ +DECLARE + v_count BIGINT; +BEGIN + IF p_dry_run THEN + SELECT count(*) INTO v_count FROM commands + WHERE status IN ('completed','failed','cancelled') + AND COALESCE(completed_at, updated_at) < NOW() - retention; + INSERT INTO cleanup_log (function_name, table_name, rows_deleted, retention, dry_run) + VALUES ('stacker_cleanup_terminal_commands', 'commands', v_count, retention, true); + + SELECT count(*) INTO v_count FROM dead_letter_queue + WHERE status IN ('exhausted','discarded','resolved') + AND updated_at < NOW() - retention; + INSERT INTO cleanup_log (function_name, table_name, rows_deleted, retention, dry_run) + VALUES ('stacker_cleanup_terminal_commands', 'dead_letter_queue', v_count, retention, true); + + SELECT count(*) INTO v_count FROM pipe_executions + WHERE completed_at < NOW() - retention; + INSERT INTO cleanup_log (function_name, table_name, rows_deleted, retention, dry_run) + VALUES ('stacker_cleanup_terminal_commands', 'pipe_executions', v_count, retention, true); + + SELECT count(*) INTO v_count FROM pipe_dag_step_executions + WHERE completed_at < NOW() - retention; + INSERT INTO cleanup_log (function_name, table_name, rows_deleted, retention, dry_run) + VALUES ('stacker_cleanup_terminal_commands', 'pipe_dag_step_executions', v_count, retention, true); + ELSE + -- Delete command_queue entries for terminal commands first (FK) + -- Use command_id (VARCHAR) not id (UUID) to match command_queue.command_id type + DELETE FROM command_queue WHERE command_id IN ( + SELECT command_id FROM commands + WHERE status IN ('completed','failed','cancelled') + AND COALESCE(completed_at, updated_at) < NOW() - retention + ); + + DELETE FROM commands + WHERE status IN ('completed','failed','cancelled') + AND COALESCE(completed_at, updated_at) < NOW() - retention; + GET DIAGNOSTICS v_count = ROW_COUNT; + INSERT INTO cleanup_log (function_name, table_name, rows_deleted, retention, dry_run) + VALUES ('stacker_cleanup_terminal_commands', 'commands', v_count, retention, false); + + DELETE FROM dead_letter_queue + WHERE status IN ('exhausted','discarded','resolved') + AND updated_at < NOW() - retention; + GET DIAGNOSTICS v_count = ROW_COUNT; + INSERT INTO cleanup_log (function_name, table_name, rows_deleted, retention, dry_run) + VALUES ('stacker_cleanup_terminal_commands', 'dead_letter_queue', v_count, retention, false); + + -- FK-safe order: step executions before executions + DELETE FROM pipe_dag_step_executions WHERE completed_at < NOW() - retention; + + DELETE FROM pipe_executions WHERE completed_at < NOW() - retention; + GET DIAGNOSTICS v_count = ROW_COUNT; + INSERT INTO cleanup_log (function_name, table_name, rows_deleted, retention, dry_run) + VALUES ('stacker_cleanup_terminal_commands', 'pipe_executions', v_count, retention, false); + END IF; +END; +$$; + + +-- 2. Add cleanup_log self-cleanup (90 days retention) +-- cleanup_log accumulates entries from all 7 cleanup functions daily but has no purge. +CREATE OR REPLACE FUNCTION stacker_cleanup_cleanup_log( + retention INTERVAL DEFAULT INTERVAL '90 days', + p_dry_run BOOLEAN DEFAULT true +) +RETURNS void LANGUAGE plpgsql AS $$ +DECLARE + v_count BIGINT; +BEGIN + IF p_dry_run THEN + SELECT count(*) INTO v_count FROM cleanup_log WHERE run_at < NOW() - retention; + INSERT INTO cleanup_log (function_name, table_name, rows_deleted, retention, dry_run) + VALUES ('stacker_cleanup_cleanup_log', 'cleanup_log', v_count, retention, true); + ELSE + DELETE FROM cleanup_log WHERE run_at < NOW() - retention; + GET DIAGNOSTICS v_count = ROW_COUNT; + INSERT INTO cleanup_log (function_name, table_name, rows_deleted, retention, dry_run) + VALUES ('stacker_cleanup_cleanup_log', 'cleanup_log', v_count, retention, false); + END IF; +END; +$$; + + +-- 3. Schedule cleanup_log purge (weekly Sunday 5:30 AM) +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_cron') THEN + IF NOT EXISTS (SELECT 1 FROM cron.job WHERE jobname = 'stacker_cleanup_cleanup_log') THEN + PERFORM cron.schedule('stacker_cleanup_cleanup_log', '30 5 * * 0', + $cron$SELECT stacker_cleanup_cleanup_log(p_dry_run := false);$cron$); + END IF; + END IF; +END; +$$; diff --git a/migrations/20260919120000_baked_snapshots_required_env_keys.down.sql b/migrations/20260919120000_baked_snapshots_required_env_keys.down.sql new file mode 100644 index 00000000..1f427e62 --- /dev/null +++ b/migrations/20260919120000_baked_snapshots_required_env_keys.down.sql @@ -0,0 +1 @@ +ALTER TABLE baked_snapshots DROP COLUMN required_env_keys; diff --git a/migrations/20260919120000_baked_snapshots_required_env_keys.up.sql b/migrations/20260919120000_baked_snapshots_required_env_keys.up.sql new file mode 100644 index 00000000..3a35e504 --- /dev/null +++ b/migrations/20260919120000_baked_snapshots_required_env_keys.up.sql @@ -0,0 +1,10 @@ +-- Record which ${VAR} references the baked compose actually needs, so the clone +-- path can refuse a deploy whose env file would not satisfy them. +-- +-- Without this, a missing key is silent: Docker Compose substitutes an empty +-- string with only a warning, the unit's ExecStartPre ends in `|| true`, and the +-- systemd unit still reports active while the stack is misconfigured. +-- +-- Nullable on purpose: snapshots baked before this column carry NULL and skip +-- the check, so existing images keep deploying unchanged. +ALTER TABLE baked_snapshots ADD COLUMN required_env_keys JSONB; diff --git a/src/bin/bake.rs b/src/bin/bake.rs index 28bf542b..547d5a96 100644 --- a/src/bin/bake.rs +++ b/src/bin/bake.rs @@ -7,7 +7,13 @@ //! Usage: //! HETZNER_TOKEN=... DATABASE_URL=... cargo run --bin bake -- \ //! --ip --stack ai-workflows-v2 --version 1.0.0 \ -//! --health-url http:///health +//! --health-url http:///health --ssh-key ~/.ssh/id_ed25519 +//! +//! `--ssh-key` is required: before snapshotting we sanitize the build box +//! (strip machine identity, blank the author's `.env`, parameterize secrets +//! embedded in compose values, drop initialized data volumes). Without it the +//! image would carry the author's credentials to every buyer, so the bake is +//! refused unless `--allow-unsanitized-snapshot` is passed deliberately. //! //! `DATABASE_URL` (the stacker Postgres) persists the BakeRecord; without it //! the bake still snapshots and prints the record, but it is not registered. @@ -23,6 +29,10 @@ async fn main() -> Result<(), Box> { let mut stack = "lamp".to_string(); let mut version = "v1".to_string(); let mut health_url: Option = None; + let mut ssh_key: Option = None; + let mut ssh_user = "root".to_string(); + let mut project_dir = "/home/trydirect/project".to_string(); + let mut allow_unsanitized = false; let mut i = 1; while i < args.len() { @@ -47,6 +57,22 @@ async fn main() -> Result<(), Box> { health_url = args.get(i + 1).cloned(); i += 2; } + "--ssh-key" => { + ssh_key = args.get(i + 1).cloned(); + i += 2; + } + "--ssh-user" => { + ssh_user = args.get(i + 1).cloned().unwrap_or(ssh_user); + i += 2; + } + "--project-dir" => { + project_dir = args.get(i + 1).cloned().unwrap_or(project_dir); + i += 2; + } + "--allow-unsanitized-snapshot" => { + allow_unsanitized = true; + i += 1; + } other => { eprintln!("ignoring unknown arg: {other}"); i += 1; @@ -77,7 +103,77 @@ async fn main() -> Result<(), Box> { let target = HetznerSnapshotTarget { provider_server_id: server_id, server_name: None, - public_ip: ip, + public_ip: ip.clone(), + }; + + // Resolve the author's field policy *before* finalizing: it decides which + // keys get blanked in `.env` and which names an embedded secret may be + // parameterized to. Also pinned to the snapshot further down so the clone + // path can regenerate those fields per buyer. + let pool = match std::env::var("DATABASE_URL") { + Ok(db_url) => Some(sqlx::PgPool::connect(&db_url).await?), + Err(_) => { + eprintln!("WARNING: DATABASE_URL not set — bake will NOT be registered in the snapshot registry."); + None + } + }; + + let config_contract = match &pool { + Some(pool) => resolve_config_contract(pool, &stack).await, + None => None, + }; + let protected_keys = config_contract + .as_ref() + .map(stacker::helpers::bake_finalize::protected_keys_from_contract) + .unwrap_or_default(); + + // Sanitize the build box before the snapshot is taken. + let finalize_outcome = match (&ssh_key, allow_unsanitized) { + (Some(key_path), _) => { + let Some(host) = ip.clone() else { + return Err("--ssh-key needs --ip (the build box address to connect to)".into()); + }; + let private_key_pem = std::fs::read_to_string(key_path) + .map_err(|e| format!("could not read --ssh-key {key_path}: {e}"))?; + + eprintln!("==> Finalizing build box before snapshot..."); + let ctx = stacker::helpers::bake_finalize::FinalizeContext { + host, + port: 22, + user: ssh_user.clone(), + private_key_pem, + project_dir: project_dir.clone(), + stack: stack.clone(), + protected_keys: protected_keys.clone(), + }; + let outcome = stacker::helpers::bake_finalize::finalize_build_box(&ctx).await?; + eprintln!( + " Sanitized. Compose requires {} env key(s): {}", + outcome.required_env_keys.len(), + outcome + .required_env_keys + .iter() + .cloned() + .collect::>() + .join(", ") + ); + Some(outcome) + } + (None, true) => { + eprintln!( + "WARNING: --allow-unsanitized-snapshot set. The image will keep the author's \ + .env values, data volumes and SSH host keys. Do NOT publish it to buyers." + ); + None + } + (None, false) => { + return Err( + "--ssh-key is required so the build box can be sanitized before \ + snapshotting (pass --allow-unsanitized-snapshot to skip, for a \ + private image only)" + .into(), + ) + } }; let connector = HetznerCloudClient::from_env().map_err(|e| e.to_string())?; @@ -94,39 +190,16 @@ async fn main() -> Result<(), Box> { ); // Persist into the snapshot registry so /api/v1/deploy/clone can resolve it. - if let Ok(db_url) = std::env::var("DATABASE_URL") { - let pool = sqlx::PgPool::connect(&db_url).await?; - - // Pin the author's field policy to this image so the clone path can - // regenerate `mutability: generated` fields fresh per buyer instead of - // shipping the single value baked into the snapshot. Resolved by the same - // slug the registry keys on (record.stack == stack_template.slug). - // Best-effort: an un-catalogued or unapproved stack bakes with no - // contract, and the clone path degrades to the baked values. - let config_contract = - match stacker::db::marketplace::get_approved_by_slug(&pool, &record.stack).await { - Ok(Some(template)) => { - match stacker::db::marketplace::get_config_contract(&pool, template.id).await { - Ok(serde_json::Value::Null) => None, - Ok(contract) => Some(contract), - Err(err) => { - eprintln!( - "WARNING: could not read config_contract for '{}': {err}", - record.stack - ); - None - } - } - } - Ok(None) => None, - Err(err) => { - eprintln!( - "WARNING: could not resolve template for '{}': {err}", - record.stack - ); - None - } - }; + if let Some(pool) = pool { + let required_env_keys = finalize_outcome.as_ref().map(|outcome| { + serde_json::Value::Array( + outcome + .required_env_keys + .iter() + .map(|key| serde_json::Value::String(key.clone())) + .collect(), + ) + }); let row = stacker::db::baked_snapshot::record( &pool, @@ -137,6 +210,7 @@ async fn main() -> Result<(), Box> { record.healthy, None, config_contract, + required_env_keys, ) .await .map_err(|e| e.to_string())?; @@ -144,9 +218,38 @@ async fn main() -> Result<(), Box> { "Registered snapshot in registry: id={} stack={}:{} image_id={}", row.id, row.stack, row.version, row.image_id ); - } else { - eprintln!("WARNING: DATABASE_URL not set — bake NOT registered in the snapshot registry."); } Ok(()) } + +/// The author's field policy for `stack`, resolved by the same slug the +/// snapshot registry keys on (`baked_snapshots.stack == stack_template.slug`). +/// +/// Best-effort: an un-catalogued or unapproved stack bakes with no contract and +/// the clone path degrades to the baked values. +async fn resolve_config_contract(pool: &sqlx::PgPool, stack: &str) -> Option { + match stacker::db::marketplace::get_approved_by_slug(pool, stack).await { + Ok(Some(template)) => { + match stacker::db::marketplace::get_config_contract(pool, template.id).await { + Ok(serde_json::Value::Null) => { + eprintln!("WARNING: template '{stack}' declares no config_contract — nothing will be regenerated per buyer."); + None + } + Ok(contract) => Some(contract), + Err(err) => { + eprintln!("WARNING: could not read config_contract for '{stack}': {err}"); + None + } + } + } + Ok(None) => { + eprintln!("WARNING: no approved template found for stack '{stack}'."); + None + } + Err(err) => { + eprintln!("WARNING: could not resolve template for '{stack}': {err}"); + None + } + } +} diff --git a/src/cli/generator/compose.rs b/src/cli/generator/compose.rs index 745a7b33..75eeb85f 100644 --- a/src/cli/generator/compose.rs +++ b/src/cli/generator/compose.rs @@ -773,6 +773,292 @@ impl fmt::Display for ComposeDefinition { } } +/// Replace literal environment values with `${VAR}` references for a set of +/// env var names. This is used during **bake** so the snapshot's compose file +/// never contains the author's secrets — Docker Compose resolves `${VAR}` from +/// the env file (`/etc/stacker/env`) at runtime. +/// +/// Only environment blocks inside service definitions are touched; other parts +/// of the compose file (labels, volumes, etc.) are left alone. +/// +/// `env_keys` is the set of env var names whose values should be +/// parameterized. Typically this is every key declared in the author's +/// `config_contract` with `mutability: generated` plus any `provided` fields. +pub fn parameterize_compose_env_vars( + compose_content: &str, + env_keys: &std::collections::HashSet, +) -> String { + if env_keys.is_empty() { + return compose_content.to_string(); + } + + let mut in_environment = false; + let mut env_indent = 0usize; + let mut result = String::with_capacity(compose_content.len()); + + for line in compose_content.lines() { + let trimmed = line.trim_start(); + let indent = line.len() - trimmed.len(); + + // Track whether we're inside an `environment:` block belonging to a + // service. The block ends when we hit a line at the same or shallower + // indent that isn't blank. + if trimmed == "environment:" { + in_environment = true; + env_indent = indent; + result.push_str(line); + result.push('\n'); + continue; + } + + if in_environment { + // A blank line or a line at the same / shallower indent ends the block. + if trimmed.is_empty() || indent <= env_indent { + in_environment = false; + } + } + + if in_environment { + // Match " KEY: value" — the key must be a valid env identifier. + if let Some((key, _rest)) = trimmed.split_once(':') { + let key = key.trim(); + if is_env_identifier(key) && env_keys.contains(key) { + // Preserve the original indent and replace the value. + let prefix = &line[..indent + key.len()]; + // Find where the value starts (after "KEY: "). + result.push_str(prefix); + result.push_str(": ${"); + result.push_str(key); + result.push_str("}\n"); + continue; + } + } + } + + result.push_str(line); + result.push('\n'); + } + + result +} + +/// Shortest value we will treat as a secret when searching *inside* other +/// values. Short strings ("admin", "postgres", a port) collide with ordinary +/// text and would corrupt the compose file. +const MIN_EMBEDDED_SECRET_LEN: usize = 12; + +/// Two protected fields resolved to the same literal value, so the compose +/// cannot be parameterized unambiguously. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EmbeddedSecretConflict { + /// The contract field names that share one value, sorted. + pub keys: Vec, +} + +impl std::fmt::Display for EmbeddedSecretConflict { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "the same secret value is declared under {} protected fields ({}). \ + Each is regenerated independently per buyer, so they would receive \ + different values and the stack would fail to authenticate. Declare \ + one field and reference it from the others.", + self.keys.len(), + self.keys.join(", ") + ) + } +} + +/// Replace secret values that appear *inside* a larger value with a `${KEY}` +/// reference — the case [`parameterize_compose_env_vars`] structurally cannot +/// reach, because it matches whole values by key name. +/// +/// The motivating case is a DSN: `DATABASE_URL: +/// postgresql://user:@host/db` carries the database password inside +/// its value, under a key name (`DATABASE_URL`) that no secret-name heuristic +/// recognises. Every existing protection layer keys off the *variable name*, so +/// such a credential is invisible to all of them at once and stays literal in +/// the baked image. +/// +/// `env_values` are the build box's resolved `KEY=value` pairs; `protected` are +/// the contract fields with `mutability: generated`/`provided`, i.e. the ones +/// that will be regenerated on the buyer's box and therefore can be referenced +/// safely. +pub fn parameterize_embedded_secret_values( + compose_content: &str, + env_values: &std::collections::BTreeMap, + protected: &std::collections::BTreeSet, +) -> Result { + // Every place a literal value is known by a name: the build box's .env plus + // the compose's own `KEY: value` pairs (a value interpolated into a service + // env block has no .env entry of its own). + let mut names_by_value: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + + for (key, value) in env_values + .iter() + .chain(compose_literal_env_pairs(compose_content).iter()) + { + if value.len() < MIN_EMBEDDED_SECRET_LEN { + continue; + } + names_by_value + .entry(value.clone()) + .or_default() + .insert(key.clone()); + } + + // A value reachable under two *protected* names diverges at regeneration + // time — refuse rather than bake an image that cannot boot. + let mut substitutions: Vec<(String, String)> = Vec::new(); + for (value, names) in &names_by_value { + let protected_names: Vec = names + .iter() + .filter(|name| protected.contains(*name)) + .cloned() + .collect(); + + match protected_names.len() { + 0 => continue, // not a managed secret; nothing regenerates it + 1 => substitutions.push((value.clone(), protected_names[0].clone())), + _ => { + return Err(EmbeddedSecretConflict { + keys: protected_names, + }) + } + } + } + + // Longest first, so a value that contains another is replaced whole. + substitutions.sort_by(|a, b| b.0.len().cmp(&a.0.len()).then(a.0.cmp(&b.0))); + + Ok(rewrite_environment_lines(compose_content, |line| { + let mut rewritten = line.to_string(); + for (value, key) in &substitutions { + if rewritten.contains(value.as_str()) { + rewritten = rewritten.replace(value.as_str(), &format!("${{{key}}}")); + } + } + rewritten + })) +} + +/// The literal `KEY: value` pairs declared in service `environment:` blocks. +fn compose_literal_env_pairs(compose_content: &str) -> std::collections::BTreeMap { + let mut pairs = std::collections::BTreeMap::new(); + + for_each_environment_line(compose_content, |line| { + if let Some((key, value)) = line.trim().split_once(':') { + let key = key.trim(); + let value = value.trim().trim_matches('"').trim_matches('\''); + if is_env_identifier(key) && !value.is_empty() && !value.starts_with("${") { + pairs.insert(key.to_string(), value.to_string()); + } + } + }); + + pairs +} + +/// Apply `rewrite` to every line inside a service `environment:` block, +/// leaving the rest of the document untouched. +fn rewrite_environment_lines( + compose_content: &str, + mut rewrite: impl FnMut(&str) -> String, +) -> String { + let mut result = String::with_capacity(compose_content.len()); + + for (line, in_environment) in environment_lines(compose_content) { + if in_environment { + result.push_str(&rewrite(line)); + } else { + result.push_str(line); + } + result.push('\n'); + } + + result +} + +fn for_each_environment_line(compose_content: &str, mut visit: impl FnMut(&str)) { + for (line, in_environment) in environment_lines(compose_content) { + if in_environment { + visit(line); + } + } +} + +/// Pair every line with whether it sits inside a service `environment:` block. +/// +/// Shares the block-tracking rule used by [`parameterize_compose_env_vars`]: +/// the block ends at the first non-blank line indented at or above the +/// `environment:` key itself. +fn environment_lines(compose_content: &str) -> Vec<(&str, bool)> { + let mut in_environment = false; + let mut env_indent = 0usize; + let mut out = Vec::new(); + + for line in compose_content.lines() { + let trimmed = line.trim_start(); + let indent = line.len() - trimmed.len(); + + if trimmed == "environment:" { + in_environment = true; + env_indent = indent; + out.push((line, false)); + continue; + } + + if in_environment && (trimmed.is_empty() || indent <= env_indent) { + in_environment = false; + } + + out.push((line, in_environment)); + } + + out +} + +/// The `${VAR}` names a compose file references, in sorted order. +/// +/// Captured at bake time and pinned to the snapshot so the clone path can +/// verify the buyer's environment satisfies the image *before* a server is +/// created. Compose resolves an unsatisfied reference to an empty string and +/// only warns, so without this check the failure surfaces as a misconfigured +/// stack rather than a refused deploy. +pub fn collect_env_var_references(compose_content: &str) -> std::collections::BTreeSet { + let mut names = std::collections::BTreeSet::new(); + let bytes = compose_content.as_bytes(); + let mut i = 0usize; + + while i + 1 < bytes.len() { + if bytes[i] != b'$' || bytes[i + 1] != b'{' { + i += 1; + continue; + } + let Some(end) = compose_content[i + 2..].find('}') else { + break; + }; + let raw = &compose_content[i + 2..i + 2 + end]; + // Compose allows ${VAR:-default} / ${VAR-default} / ${VAR:?err}. + let name = raw.split([':', '-', '?', '+']).next().unwrap_or("").trim(); + if is_env_identifier(name) { + names.insert(name.to_string()); + } + i += 2 + end + 1; + } + + names +} + +/// Returns `true` when `s` looks like a POSIX env-variable name. +fn is_env_identifier(s: &str) -> bool { + !s.is_empty() + && s.chars() + .enumerate() + .all(|(i, c)| c.is_ascii_alphanumeric() || c == '_' || (i == 0 && c == '.')) +} + // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ // Tests // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ @@ -1861,4 +2147,222 @@ services: output ); } + + #[test] + fn parameterize_replaces_secret_values_with_env_refs() { + let compose = "\ +services: + app: + image: trydirect/stackpilot:latest + ports: + - \"8080:8000\" + environment: + ADMIN_PASSWORD: 4f4237dd9bfe8e1622706cac7bab63c7 + ADMIN_USER: admin + DATABASE_URL: postgresql://stackpilot:2213a996143863b99a0f2d3e22907690@db:5432/stackpilot + SECRET_KEY: b838f1f22379b8c268a4d3e0268459761c18947e1576956a6d9b1f3928070df4 + OLLAMA_MODEL: llama3.1 + restart: unless-stopped +"; + let mut keys = std::collections::HashSet::new(); + keys.insert("ADMIN_PASSWORD".to_string()); + keys.insert("SECRET_KEY".to_string()); + keys.insert("DATABASE_URL".to_string()); + + let result = parameterize_compose_env_vars(compose, &keys); + + assert!( + result.contains("ADMIN_PASSWORD: ${ADMIN_PASSWORD}"), + "admin pw:\n{result}" + ); + assert!( + result.contains("SECRET_KEY: ${SECRET_KEY}"), + "secret key:\n{result}" + ); + assert!( + result.contains("DATABASE_URL: ${DATABASE_URL}"), + "db url:\n{result}" + ); + // Non-secret values stay as-is. + assert!( + result.contains("ADMIN_USER: admin"), + "admin user:\n{result}" + ); + assert!( + result.contains("OLLAMA_MODEL: llama3.1"), + "model:\n{result}" + ); + } + + #[test] + fn parameterize_leaves_non_env_blocks_alone() { + let compose = "\ +services: + app: + image: myapp:latest + environment: + SECRET_KEY: abc123 + labels: + my.stacker.service: myapp + volumes: + - app_data:/app/data +"; + let mut keys = std::collections::HashSet::new(); + keys.insert("SECRET_KEY".to_string()); + + let result = parameterize_compose_env_vars(compose, &keys); + + assert!( + result.contains("SECRET_KEY: ${SECRET_KEY}"), + "secret:\n{result}" + ); + assert!( + result.contains("my.stacker.service: myapp"), + "label:\n{result}" + ); + assert!(result.contains("- app_data:/app/data"), "volume:\n{result}"); + } + + // ── embedded secret values (secrets inside a larger value) ────────────── + + fn env_map(pairs: &[(&str, &str)]) -> std::collections::BTreeMap { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + fn key_set(keys: &[&str]) -> std::collections::BTreeSet { + keys.iter().map(|k| k.to_string()).collect() + } + + #[test] + fn embedded_secret_inside_a_dsn_is_parameterized() { + // The case whole-value replacement structurally cannot reach. + let compose = "\ +services: + app: + environment: + DATABASE_URL: postgresql://stackpilot:2213a996143863b99a0f2d3e22907690@db:5432/stackpilot +"; + let env = env_map(&[("POSTGRES_PASSWORD", "2213a996143863b99a0f2d3e22907690")]); + let result = + parameterize_embedded_secret_values(compose, &env, &key_set(&["POSTGRES_PASSWORD"])) + .expect("no conflict"); + + assert!( + result.contains("postgresql://stackpilot:${POSTGRES_PASSWORD}@db:5432/stackpilot"), + "password replaced in place:\n{result}" + ); + assert!( + !result.contains("2213a996143863b99a0f2d3e22907690"), + "no literal left:\n{result}" + ); + } + + #[test] + fn short_values_are_left_alone() { + // "admin" would otherwise corrupt every word containing it. + let compose = "services:\n app:\n environment:\n ADMIN_USER: admin\n GREETING: administrator\n"; + let env = env_map(&[("ADMIN_USER", "admin")]); + let result = parameterize_embedded_secret_values(compose, &env, &key_set(&["ADMIN_USER"])) + .expect("no conflict"); + assert!( + result.contains("GREETING: administrator"), + "untouched:\n{result}" + ); + } + + #[test] + fn undeclared_values_are_left_alone() { + // Nothing regenerates it on the buyer's box, so a ${REF} would resolve empty. + let compose = + "services:\n app:\n environment:\n URL: http://host/aaaaaaaaaaaaaaaa\n"; + let env = env_map(&[("SOME_KEY", "aaaaaaaaaaaaaaaa")]); + let result = + parameterize_embedded_secret_values(compose, &env, &std::collections::BTreeSet::new()) + .expect("no conflict"); + assert!( + result.contains("aaaaaaaaaaaaaaaa"), + "left literal:\n{result}" + ); + } + + #[test] + fn one_value_under_two_protected_names_is_refused() { + // stackpilot's DB_PASSWORD/POSTGRES_PASSWORD duplication: both are + // `generated`, so regeneration would hand them different values. + let compose = "\ +services: + db: + environment: + POSTGRES_PASSWORD: 2213a996143863b99a0f2d3e22907690 +"; + let env = env_map(&[("DB_PASSWORD", "2213a996143863b99a0f2d3e22907690")]); + let err = parameterize_embedded_secret_values( + compose, + &env, + &key_set(&["DB_PASSWORD", "POSTGRES_PASSWORD"]), + ) + .expect_err("duplicate must be refused"); + + assert_eq!( + err.keys, + vec!["DB_PASSWORD".to_string(), "POSTGRES_PASSWORD".to_string()] + ); + } + + #[test] + fn non_environment_blocks_are_never_rewritten() { + let compose = "\ +services: + app: + image: myapp:2213a996143863b99a0f2d3e22907690 + environment: + TOKEN: 2213a996143863b99a0f2d3e22907690 +"; + let env = env_map(&[("TOKEN", "2213a996143863b99a0f2d3e22907690")]); + let result = parameterize_embedded_secret_values(compose, &env, &key_set(&["TOKEN"])) + .expect("no conflict"); + + assert!( + result.contains("image: myapp:2213a996143863b99a0f2d3e22907690"), + "image digest untouched:\n{result}" + ); + assert!( + result.contains("TOKEN: ${TOKEN}"), + "env replaced:\n{result}" + ); + } + + #[test] + fn collects_env_var_references_with_defaults_and_ignores_literals() { + let compose = "\ +services: + app: + image: app:latest + environment: + A: ${ALPHA} + B: ${BETA:-fallback} + C: ${GAMMA?required} + D: plain-value +"; + let refs = collect_env_var_references(compose); + let found: Vec<&str> = refs.iter().map(String::as_str).collect(); + assert_eq!(found, vec!["ALPHA", "BETA", "GAMMA"]); + } + + #[test] + fn collects_env_var_reference_embedded_in_a_dsn() { + let compose = + "services:\n app:\n environment:\n DATABASE_URL: postgres://u:${PW}@h/db\n"; + assert!(collect_env_var_references(compose).contains("PW")); + } + + #[test] + fn parameterize_no_keys_returns_original() { + let compose = "services:\n app:\n environment:\n FOO: bar\n"; + let keys = std::collections::HashSet::new(); + assert_eq!(parameterize_compose_env_vars(compose, &keys), compose); + } } diff --git a/src/cli/stacker_client.rs b/src/cli/stacker_client.rs index 340a2a1f..ce05dd3b 100644 --- a/src/cli/stacker_client.rs +++ b/src/cli/stacker_client.rs @@ -3609,6 +3609,41 @@ impl StackerClient { Ok(()) } + + /// Resubmit an approved/rejected/needs_changes template with a new version. + pub async fn marketplace_resubmit( + &self, + template_id: &str, + body: serde_json::Value, + ) -> Result<(), CliError> { + let url = format!("{}/api/templates/{}/resubmit", self.base_url, template_id); + let resp = self + .http + .post(&url) + .bearer_auth(&self.token) + .json(&body) + .send() + .await + .map_err(|e| { + CliError::MarketplaceFailed(format!("Stacker server unreachable: {}", e)) + })?; + + if !resp.status().is_success() { + let status = resp.status().as_u16(); + let body = resp.text().await.unwrap_or_default(); + return Err(CliError::MarketplaceFailed( + stacker_api_failure_with_message( + "Resubmit failed", + &format!("POST /api/templates/{template_id}/resubmit"), + status, + &body, + cli_debug_enabled(), + ), + )); + } + + Ok(()) + } } // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ @@ -4061,6 +4096,21 @@ pub fn build_project_body(config: &StackerConfig) -> serde_json::Value { service_apps.push(service_to_app_json(svc, &network_ids)); } + // `project_app.config_contract` is persisted by a separate server-side + // accessor. Include the full contract on each generated app so `stacker + // sync` does not silently discard the policy from stacker.yml. + if let Ok(config_contract) = serde_json::to_value(&config.config_contract) { + let has_services = config_contract + .get("services") + .and_then(serde_json::Value::as_object) + .is_some_and(|services| !services.is_empty()); + if has_services { + for app in web_apps.iter_mut().chain(service_apps.iter_mut()) { + app["config_contract"] = config_contract.clone(); + } + } + } + serde_json::json!({ "custom": { "custom_stack_code": stack_code, @@ -5094,6 +5144,40 @@ mod tests { ); } + #[test] + fn build_project_body_includes_config_contract_on_apps() { + let mut config = crate::cli::config_parser::ConfigBuilder::new() + .name("contract-project") + .app_image("nginx:1.27") + .build() + .expect("config should build"); + config.config_contract = serde_json::from_value(serde_json::json!({ + "services": { + "app": { + "fields": { + "JWT_SECRET": { + "mutability": "generated", + "type": "hex", + "length": 32 + } + } + } + } + })) + .expect("config contract should deserialize"); + + let body = build_project_body(&config); + let contract = &body["custom"]["web"][0]["config_contract"]; + assert_eq!( + contract["services"]["app"]["fields"]["JWT_SECRET"]["mutability"], + "generated" + ); + assert_eq!( + contract["services"]["app"]["fields"]["JWT_SECRET"]["type"], + "hex" + ); + } + #[test] fn pipe_instance_request_serializes_adapter_references() { let request = CreatePipeInstanceApiRequest { diff --git a/src/console/commands/cli/agent.rs b/src/console/commands/cli/agent.rs index 6e18d438..d0ad16df 100644 --- a/src/console/commands/cli/agent.rs +++ b/src/console/commands/cli/agent.rs @@ -1754,11 +1754,15 @@ impl CallableTrait for AgentStatusCommand { n_apps, ), ); - let live_containers = match fetch_live_containers(&ctx, &hash) { - Ok(list) => list, - Err(err) => { - eprintln!("Warning: failed to fetch live containers: {}", err); - None + let live_containers = if agent_status == "offline" { + None + } else { + match fetch_live_containers(&ctx, &hash) { + Ok(list) => list, + Err(err) => { + eprintln!("Warning: failed to fetch live containers: {}", err); + None + } } }; @@ -2260,10 +2264,32 @@ fn run_logs_command( ) } +/// Returns `true` if the agent for the given deployment is offline. +/// +/// Fetches a lightweight snapshot to check status before attempting +/// any agent commands that would hang waiting for an unreachable agent. +fn is_agent_offline(ctx: &CliRuntime, deployment_hash: &str) -> bool { + let snapshot = ctx.block_on(ctx.client.agent_snapshot(deployment_hash)); + match snapshot { + Ok(snap) => { + let item = snap.get("item").unwrap_or(&snap); + item.get("agent") + .and_then(|a| a.get("status")) + .and_then(|s| s.as_str()) + == Some("offline") + } + Err(_) => false, // If we can't fetch snapshot, let the caller try + } +} + pub(crate) fn fetch_live_containers( ctx: &CliRuntime, deployment_hash: &str, ) -> Result>, CliError> { + if is_agent_offline(ctx, deployment_hash) { + return Ok(None); + } + let params = crate::forms::status_panel::ListContainersCommandRequest { include_health: true, include_logs: false, @@ -4341,6 +4367,71 @@ monitoring: None => std::env::remove_var("STACKER_DOCKER_REGISTRY"), } } + + // ── Agent status snapshot parsing ──────────────────────────────────── + + #[test] + fn extract_agent_status_from_snapshot_offline() { + let snap = serde_json::json!({ + "item": { + "agent": { "status": "offline", "version": "1.0.0" }, + "apps": [] + } + }); + let item = snapshot_item(&snap); + let status = item + .get("agent") + .and_then(|a| a.get("status")) + .and_then(|s| s.as_str()) + .unwrap_or("unknown"); + assert_eq!(status, "offline"); + } + + #[test] + fn extract_agent_status_from_snapshot_online() { + let snap = serde_json::json!({ + "item": { + "agent": { "status": "online", "version": "1.0.0" }, + "apps": [] + } + }); + let item = snapshot_item(&snap); + let status = item + .get("agent") + .and_then(|a| a.get("status")) + .and_then(|s| s.as_str()) + .unwrap_or("unknown"); + assert_eq!(status, "online"); + } + + #[test] + fn extract_agent_status_missing_defaults_unknown() { + let snap = serde_json::json!({ "item": { "apps": [] } }); + let item = snapshot_item(&snap); + let status = item + .get("agent") + .and_then(|a| a.get("status")) + .and_then(|s| s.as_str()) + .unwrap_or("unknown"); + assert_eq!(status, "unknown"); + } + + #[test] + fn snapshot_item_prefers_item_key() { + let snap = serde_json::json!({ + "item": { "agent": { "status": "online" } }, + "agent": { "status": "offline" } + }); + let item = snapshot_item(&snap); + assert_eq!(item["agent"]["status"], "online"); + } + + #[test] + fn snapshot_item_falls_back_to_root() { + let snap = serde_json::json!({ "agent": { "status": "offline" } }); + let item = snapshot_item(&snap); + assert_eq!(item["agent"]["status"], "offline"); + } } // ── rotate-token ──────────────────────────────────────────────────────────── diff --git a/src/console/commands/cli/deploy.rs b/src/console/commands/cli/deploy.rs index 2c9ff383..37481d7d 100644 --- a/src/console/commands/cli/deploy.rs +++ b/src/console/commands/cli/deploy.rs @@ -686,6 +686,32 @@ fn normalize_generated_compose_paths(compose_path: &Path) -> Result<(), CliError Ok(()) } +fn compose_env_keys(config: &StackerConfig) -> std::collections::HashSet { + let mut keys: std::collections::HashSet = config.env.keys().cloned().collect(); + + // Include policy-declared fields even when they are defined only in + // app.environment or services[].environment rather than top-level env. + if let Ok(contract) = serde_json::to_value(&config.config_contract) { + if let Some(services) = contract.get("services").and_then(|v| v.as_object()) { + for service in services.values() { + if let Some(fields) = service.get("fields").and_then(|v| v.as_object()) { + for (name, policy) in fields { + let protected = matches!( + policy.get("mutability").and_then(|v| v.as_str()), + Some("generated") | Some("provided") + ); + if protected { + keys.insert(name.clone()); + } + } + } + } + } + } + + keys +} + /// A compose service that declares a `build:` section. struct ComposeBuildService { name: String, @@ -3556,7 +3582,20 @@ fn run_deploy_with_credentials_manager( let compose = ComposeDefinition::try_from(&config)?; // `write_to` refuses to clobber an existing file unless told to, // so a staleness-driven regeneration must opt in explicitly. - compose.write_to(&compose_out, force_rebuild || compose_is_stale)?; + // Parameterize secret env vars: replace literal values with + // `${VAR}` references so the compose file never contains the + // author's secrets. Docker Compose resolves them from the + // co-located `.env` file at runtime. + let rendered = compose.render(); + let env_keys = compose_env_keys(&config); + let parameterized = + crate::cli::generator::compose::parameterize_compose_env_vars( + &rendered, + &env_keys, + ); + if force_rebuild || compose_is_stale || !compose_out.exists() { + std::fs::write(&compose_out, ¶meterized)?; + } // The synthesized caddy/nginx proxy service mounts a config file // (./Caddyfile, ./nginx/conf.d) from the compose directory. For // local/server deploys the tfa proxy role does NOT run, so the @@ -3575,6 +3614,21 @@ fn run_deploy_with_credentials_manager( (compose_out, false) }; + // Parameterize an existing generated compose file as well. This prevents + // a previously rendered file with literal secrets from bypassing the + // protection merely because it was considered up to date. + if !compose_is_user_supplied { + let env_keys = compose_env_keys(&config); + if !env_keys.is_empty() { + let content = std::fs::read_to_string(&compose_path)?; + let parameterized = + crate::cli::generator::compose::parameterize_compose_env_vars(&content, &env_keys); + if parameterized != content { + std::fs::write(&compose_path, parameterized)?; + } + } + } + normalize_generated_compose_paths(&compose_path)?; validate_compose_for_deploy(&compose_path)?; reject_build_sections_for_cloud( diff --git a/src/console/commands/cli/submit.rs b/src/console/commands/cli/submit.rs index de3608f8..9aa1932a 100644 --- a/src/console/commands/cli/submit.rs +++ b/src/console/commands/cli/submit.rs @@ -153,11 +153,22 @@ impl CallableTrait for SubmitCommand { // Create or update the template on the server eprintln!("Creating/updating template '{}'...", name); - let template = client.marketplace_create_or_update(body).await?; - - // Submit for review - eprintln!("Submitting for marketplace review..."); - client.marketplace_submit(&template.id).await?; + let template = client.marketplace_create_or_update(body.clone()).await?; + + // Submit for review — use resubmit endpoint for templates that + // are already approved (submit_for_review only allows + // draft/rejected/needs_changes). + if template.status == "approved" { + eprintln!("Resubmitting approved template for marketplace review..."); + let mut resubmit_body = body.clone(); + resubmit_body["confirm_no_secrets"] = serde_json::json!(true); + client + .marketplace_resubmit(&template.id, resubmit_body) + .await?; + } else { + eprintln!("Submitting for marketplace review..."); + client.marketplace_submit(&template.id).await?; + } // Success message println!(); diff --git a/src/db/agent.rs b/src/db/agent.rs index 16cb9f2f..260a6dc1 100644 --- a/src/db/agent.rs +++ b/src/db/agent.rs @@ -239,6 +239,75 @@ pub async fn delete(pool: &PgPool, agent_id: Uuid) -> Result<(), String> { }) } +/// Delete agents whose deployment is gone (or soft-deleted) and that show no +/// sign of life within the retention window. +/// +/// "No sign of life" means both: +/// - `last_heartbeat` is NULL or older than the window, AND +/// - no `audit_log` row references the agent (by id or deployment_hash) +/// within the same window. +/// +/// The second condition protects agents that are alive but failing +/// authentication — `last_heartbeat` only advances on successful `wait`/`report`, +/// while `audit_log` captures `auth_failure` entries. +#[tracing::instrument(name = "Sweep dead agents", skip(pool))] +pub async fn sweep_dead(pool: &PgPool, retention_days: i32) -> Result { + let result = sqlx::query( + r#" + DELETE FROM agents a + WHERE NOT EXISTS ( + SELECT 1 FROM deployment d + WHERE d.deployment_hash = a.deployment_hash + AND d.deleted IS NOT TRUE + ) + AND (a.last_heartbeat IS NULL + OR a.last_heartbeat < NOW() - make_interval(days => $1)) + AND NOT EXISTS ( + SELECT 1 FROM audit_log l + WHERE (l.agent_id = a.id OR l.deployment_hash = a.deployment_hash) + AND l.created_at > NOW() - make_interval(days => $1) + ) + "#, + ) + .bind(retention_days) + .execute(pool) + .await + .map_err(|err| { + tracing::error!("Failed to sweep dead agents: {:?}", err); + format!("Database error: {}", err) + })?; + + Ok(result.rows_affected()) +} + +/// Delete agents whose `deployment_hash` is structurally invalid. +/// +/// A valid hash matches `deployment_` (36-char UUID with hyphens). +/// Rows with an invalid hash can never be matched by their own agent — the +/// lookup in `fetch_by_deployment_hash` will never find them. Among the 18 +/// currently broken rows, 17 store a raw agent token (86-char base64url) +/// instead of a hash, leaking the secret in plaintext. +/// +/// Returns its own count separate from `sweep_dead` so the caller can log +/// exactly how many malformed rows were removed. +#[tracing::instrument(name = "Sweep malformed agent rows", skip(pool))] +pub async fn sweep_malformed(pool: &PgPool) -> Result { + let result = sqlx::query( + r#" + DELETE FROM agents + WHERE deployment_hash !~ '^deployment_[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$' + "#, + ) + .execute(pool) + .await + .map_err(|err| { + tracing::error!("Failed to sweep malformed agents: {:?}", err); + format!("Database error: {}", err) + })?; + + Ok(result.rows_affected()) +} + pub async fn log_audit( pool: &PgPool, audit_log: models::AuditLog, diff --git a/src/db/baked_snapshot.rs b/src/db/baked_snapshot.rs index 7aed2106..160839f2 100644 --- a/src/db/baked_snapshot.rs +++ b/src/db/baked_snapshot.rs @@ -64,11 +64,12 @@ pub async fn record( healthy: bool, digests: Option, config_contract: Option, + required_env_keys: Option, ) -> Result { sqlx::query_as::<_, BakedSnapshot>( r#" - INSERT INTO baked_snapshots (stack, version, provider, image_id, healthy, digests, config_contract) - VALUES ($1, $2, $3, $4, $5, $6, $7) + INSERT INTO baked_snapshots (stack, version, provider, image_id, healthy, digests, config_contract, required_env_keys) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING * "#, ) @@ -79,6 +80,7 @@ pub async fn record( .bind(healthy) .bind(digests) .bind(config_contract) + .bind(required_env_keys) .fetch_one(pool) .await .map_err(|e| format!("Failed to record baked snapshot: {e}")) diff --git a/src/db/marketplace.rs b/src/db/marketplace.rs index 2e05627e..adf0a6cb 100644 --- a/src/db/marketplace.rs +++ b/src/db/marketplace.rs @@ -686,6 +686,10 @@ pub async fn set_source_project_id( } /// Read back the source project linked via [`set_source_project_id`]. +/// Checks the latest version first, then falls back to any version — +/// `source_project_id` is semantically tied to the template, not a +/// specific version, and `resubmit_with_new_version` may create a new +/// latest row before the caller has a chance to re-set it. pub async fn get_source_project_id( pool: &PgPool, template_id: uuid::Uuid, @@ -696,7 +700,9 @@ pub async fn get_source_project_id( sqlx::query_scalar::<_, Option>( r#"SELECT source_project_id FROM stack_template_version - WHERE template_id = $1 AND is_latest = true + WHERE template_id = $1 + AND source_project_id IS NOT NULL + ORDER BY is_latest DESC, created_at DESC LIMIT 1"#, ) .bind(template_id) diff --git a/src/forms/project/app.rs b/src/forms/project/app.rs index 6387666f..b176fee9 100644 --- a/src/forms/project/app.rs +++ b/src/forms/project/app.rs @@ -72,6 +72,11 @@ pub struct App { pub network: forms::project::ServiceNetworks, #[validate] pub shared_ports: Option>, + /// Author-declared per-field policy (`config_contract` from stacker.yml). + /// Persisted to `project_app.config_contract` during sync so the Stack + /// Builder UI can render form fields with proper generation/validation rules. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub config_contract: Option, } impl App { diff --git a/src/helpers/bake.rs b/src/helpers/bake.rs index 658446a1..7503890e 100644 --- a/src/helpers/bake.rs +++ b/src/helpers/bake.rs @@ -35,6 +35,11 @@ pub enum BakeError { NoImageId, #[error("bake backend error: {0}")] Backend(String), + /// The build box could not be sanitized before snapshotting. Refused rather + /// than published: an unsanitized image carries the author's credentials to + /// every buyer. + #[error("bake rejected: could not finalize build box: {0}")] + Finalize(String), } /// Pure gate: only a healthy build box with a real `image_id` yields a record. diff --git a/src/helpers/bake_finalize.rs b/src/helpers/bake_finalize.rs new file mode 100644 index 00000000..56eb5c89 --- /dev/null +++ b/src/helpers/bake_finalize.rs @@ -0,0 +1,434 @@ +//! Finalize a build box before it is snapshotted (immutable-deploy BAKE step). +//! +//! A bake snapshots the *whole disk* of a build box that the author deployed +//! normally. That disk carries three classes of the author's own secrets, none +//! of which a buyer's clone should inherit: +//! +//! 1. **Machine identity** — SSH host keys and `machine-id`. Left in place, +//! every clone of the image shares them, so any buyer can impersonate the +//! SSH host of any other buyer. This is the standard "sysprep" step every +//! golden-image pipeline performs (`virt-sysprep`, Packer, the DigitalOcean +//! 1-Click checklist); we had none. +//! 2. **The co-located `.env`** — shipped verbatim next to the compose file by +//! the deploy bundle, so the author's literal secrets sit in the image even +//! when the compose itself is fully parameterized. Parameterizing compose +//! alone just moves the secret from one file in the image to another. +//! 3. **Initialized data volumes** — a secret the app wrote into its own +//! database/volume on first run is frozen in the snapshot and is *not* +//! governed by environment variables any more. Postgres is the canonical +//! case: `POSTGRES_PASSWORD` is honoured only when it initializes an empty +//! data directory, so a cloned box silently keeps the author's role password. +//! +//! Everything here is a pure function producing shell, so the policy is +//! unit-tested without infra; [`crate::helpers::bake`] wires it to a real SSH +//! session. + +use std::collections::BTreeSet; + +/// Volumes whose content must survive the bake, keyed by stack slug. +/// +/// Defaulting to *reset* is deliberate: wrongly resetting a volume costs a +/// rebuild of cheap state, while wrongly keeping one leaks the author's +/// credentials to every buyer. Only volumes that are expensive to rebuild +/// **and** carry no credentials belong here. +/// +/// `stackpilot`'s Ollama volume holds the pulled model weights — gigabytes, +/// with a 600s pull timeout in `scripts/download-model.sh`. Preserving it is +/// the entire economic point of baking that stack. +pub fn volumes_to_keep(stack: &str) -> &'static [&'static str] { + match stack { + "stackpilot" => &["ollama"], + _ => &[], + } +} + +/// Shell to strip machine identity so each clone boots as a distinct host. +/// +/// `sshd` regenerates host keys on first boot when none are present, and +/// `systemd` repopulates an empty `/etc/machine-id`; clearing cloud-init's +/// instance state makes it treat the clone as a new instance and re-run its +/// per-instance modules. +pub fn identity_reset_commands() -> Vec { + vec![ + "rm -f /etc/ssh/ssh_host_*".to_string(), + ": > /etc/machine-id".to_string(), + "rm -f /var/lib/dbus/machine-id".to_string(), + "rm -rf /var/lib/cloud/instances /var/lib/cloud/instance".to_string(), + "rm -f /root/.bash_history /home/*/.bash_history".to_string(), + "find /var/log -type f -exec truncate -s 0 {} + 2>/dev/null || true".to_string(), + ] +} + +/// Shell to stop the stack and drop every data volume that is not explicitly +/// preserved, so the buyer's box initializes them from scratch with the +/// buyer's own generated values. +/// +/// Volume names are matched by suffix because Compose prefixes them with the +/// project name (`project_stackpilot_pgdata` for a declared `stackpilot_pgdata`). +pub fn volume_reset_commands(project_dir: &str, keep: &[&str]) -> Vec { + let mut cmds = vec![format!( + "cd {project_dir} && docker compose down --remove-orphans" + )]; + + let filter = if keep.is_empty() { + "cat".to_string() + } else { + let pattern = keep.join("|"); + format!("grep -Ev '({pattern})'") + }; + + cmds.push(format!( + "docker volume ls -q | {filter} | xargs -r docker volume rm -f" + )); + cmds +} + +/// Blank the values of secret-bearing keys in a `.env` file while keeping the +/// file's shape: keys, comments, blank lines and non-secret values survive. +/// +/// The keys are kept (rather than the lines dropped) so the file still +/// documents what the stack expects; on the buyer's box the whole file is +/// replaced from `/etc/stacker/env` by the systemd unit's `ExecStartPre`, so +/// the blanked values are never read. +pub fn scrub_env_file(content: &str, protected: &BTreeSet) -> String { + let mut out = String::with_capacity(content.len()); + + for line in content.lines() { + let trimmed = line.trim_start(); + if trimmed.is_empty() || trimmed.starts_with('#') { + out.push_str(line); + out.push('\n'); + continue; + } + + match line.split_once('=') { + Some((key, _value)) if should_blank(key.trim(), protected) => { + out.push_str(key); + out.push_str("=\n"); + } + _ => { + out.push_str(line); + out.push('\n'); + } + } + } + + out +} + +/// A key is blanked when the author's contract declared it regenerable/buyer- +/// supplied, or when its name is secret-shaped by the same heuristic the CLI +/// already uses for `generate-secrets.sh`. +fn should_blank(key: &str, protected: &BTreeSet) -> bool { + protected.contains(key) || crate::console::commands::cli::init::is_secret_env_key(key) +} + +/// Everything the finalize step needs to reach and sanitize a build box. +#[derive(Debug, Clone)] +pub struct FinalizeContext { + pub host: String, + pub port: u16, + pub user: String, + pub private_key_pem: String, + /// Where the deploy put the compose file and its co-located `.env`. + pub project_dir: String, + /// Stack slug — selects the volume keep-list. + pub stack: String, + /// Contract fields with `mutability: generated`/`provided`. + pub protected_keys: BTreeSet, +} + +/// What the finalize step learned about the image it just sanitized. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FinalizeOutcome { + /// `${VAR}` names the sanitized compose references — pinned to the snapshot + /// so the clone path can fail closed on an environment that cannot satisfy + /// them. + pub required_env_keys: BTreeSet, +} + +/// Sanitize a build box in place, immediately before it is snapshotted. +/// +/// Order matters: the compose/env rewrite has to happen while the stack is +/// still described on disk, the volume reset tears the stack down, and the +/// identity reset goes last because it leaves the box unable to present a +/// stable SSH identity afterwards. The build box is throwaway, so none of this +/// needs to be reversible. +pub async fn finalize_build_box( + ctx: &FinalizeContext, +) -> Result { + use crate::helpers::bake::BakeError; + use crate::helpers::ssh_client::{disconnect_ssh, exec_remote, open_ssh}; + + let fail = |stage: &str, err: String| BakeError::Finalize(format!("{stage}: {err}")); + + let session = open_ssh( + &ctx.host, + ctx.port, + &ctx.user, + &ctx.private_key_pem, + std::time::Duration::from_secs(30), + ) + .await + .map_err(|e| fail("ssh connect", e.to_string()))?; + + let run = |cmd: String| { + let session = &session; + async move { + let (stdout, stderr, code) = exec_remote(session, &cmd, 300) + .await + .map_err(|e| e.to_string())?; + if code != 0 { + return Err(format!("`{cmd}` exited {code}: {stderr}")); + } + Ok::(stdout) + } + }; + + let compose_path = format!("{}/docker-compose.yml", ctx.project_dir); + let env_path = format!("{}/.env", ctx.project_dir); + + let result = async { + // 1. Read what the deploy left on the box. + let compose = run(format!("cat {compose_path}")) + .await + .map_err(|e| fail("read compose", e))?; + // A stack may legitimately have no .env; treat that as empty. + let env_raw = run(format!("cat {env_path} 2>/dev/null || true")) + .await + .map_err(|e| fail("read .env", e))?; + let env_values = parse_env_pairs(&env_raw); + + // 2. Replace secrets embedded inside larger values (DSNs) with ${KEY} + // references. Whole-value keys were already parameterized at deploy + // time by `parameterize_compose_env_vars`. + let sanitized = crate::cli::generator::compose::parameterize_embedded_secret_values( + &compose, + &env_values, + &ctx.protected_keys, + ) + .map_err(|conflict| BakeError::Finalize(conflict.to_string()))?; + + if sanitized != compose { + write_remote_file(&run, &compose_path, &sanitized) + .await + .map_err(|e| fail("write compose", e))?; + } + + // 3. Record what the image now needs from the buyer's env file. + let required_env_keys = + crate::cli::generator::compose::collect_env_var_references(&sanitized); + + // 4. Blank the author's secrets in the co-located .env. The buyer's box + // overwrites this file wholesale from /etc/stacker/env at boot, so + // the blanked values are never read. + if !env_raw.trim().is_empty() { + let scrubbed = scrub_env_file(&env_raw, &ctx.protected_keys); + write_remote_file(&run, &env_path, &scrubbed) + .await + .map_err(|e| fail("write .env", e))?; + } + + // 5. Drop initialized data volumes so the buyer's box re-initializes + // them with the buyer's own values. A secret the app persisted on + // first run is not governed by env vars any more. + for cmd in volume_reset_commands(&ctx.project_dir, volumes_to_keep(&ctx.stack)) { + run(cmd).await.map_err(|e| fail("volume reset", e))?; + } + + // 6. Strip machine identity last. + for cmd in identity_reset_commands() { + run(cmd).await.map_err(|e| fail("identity reset", e))?; + } + + Ok(FinalizeOutcome { required_env_keys }) + } + .await; + + disconnect_ssh(session).await; + result +} + +/// Write `content` to `path` on the remote box without any quoting hazards: +/// the payload travels base64-encoded and is decoded on the far side. +async fn write_remote_file(run: &F, path: &str, content: &str) -> Result<(), String> +where + F: Fn(String) -> Fut, + Fut: std::future::Future>, +{ + use base64::{engine::general_purpose::STANDARD, Engine as _}; + let encoded = STANDARD.encode(content.as_bytes()); + run(format!("printf %s {encoded} | base64 -d > {path}")) + .await + .map(|_| ()) +} + +/// Parse `KEY=value` lines into a map, skipping comments and blanks. +pub fn parse_env_pairs(content: &str) -> std::collections::BTreeMap { + let mut pairs = std::collections::BTreeMap::new(); + + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + if let Some((key, value)) = trimmed.split_once('=') { + let value = value.trim(); + if !value.is_empty() { + pairs.insert(key.trim().to_string(), value.to_string()); + } + } + } + + pairs +} + +/// The contract fields a buyer's box regenerates or supplies — i.e. the ones a +/// `${KEY}` reference can safely point at. +/// +/// Mirrors the selection `compose_env_keys` makes at deploy time +/// (`src/console/commands/cli/deploy.rs`), reading the contract as raw JSON so +/// an unparseable or partially-shaped contract degrades to "nothing protected" +/// rather than failing the bake outright. +pub fn protected_keys_from_contract(contract: &serde_json::Value) -> BTreeSet { + let mut keys = BTreeSet::new(); + + let Some(services) = contract.get("services").and_then(|v| v.as_object()) else { + return keys; + }; + + for service in services.values() { + let Some(fields) = service.get("fields").and_then(|v| v.as_object()) else { + continue; + }; + for (name, policy) in fields { + let protected = matches!( + policy.get("mutability").and_then(|v| v.as_str()), + Some("generated") | Some("provided") + ); + if protected { + keys.insert(name.clone()); + } + } + } + + keys +} + +#[cfg(test)] +mod tests { + use super::*; + + fn protected(keys: &[&str]) -> BTreeSet { + keys.iter().map(|k| k.to_string()).collect() + } + + #[test] + fn scrub_blanks_contract_declared_keys() { + let env = "SECRET_KEY=b838f1f2\nOLLAMA_MODEL=llama3.1\n"; + let out = scrub_env_file(env, &protected(["SECRET_KEY"].as_slice())); + assert!(out.contains("SECRET_KEY=\n"), "blanked:\n{out}"); + assert!( + out.contains("OLLAMA_MODEL=llama3.1"), + "non-secret kept:\n{out}" + ); + } + + #[test] + fn scrub_blanks_secret_shaped_keys_without_a_contract() { + // No contract at all — the name heuristic still has to catch these. + let env = "DB_PASSWORD=2213a996\nADMIN_USER=admin\n"; + let out = scrub_env_file(env, &BTreeSet::new()); + assert!(out.contains("DB_PASSWORD=\n"), "blanked:\n{out}"); + assert!(out.contains("ADMIN_USER=admin"), "non-secret kept:\n{out}"); + } + + #[test] + fn scrub_preserves_comments_and_blank_lines() { + let env = "# Secrets\n\nSECRET_KEY=abc\n"; + let out = scrub_env_file(env, &BTreeSet::new()); + assert_eq!(out, "# Secrets\n\nSECRET_KEY=\n"); + } + + #[test] + fn scrub_keeps_values_containing_equals_signs() { + // A blanked key must not be confused by '=' inside a kept value. + let env = "OLLAMA_MODEL=llama3.1\nJWT_SECRET=a=b=c\n"; + let out = scrub_env_file(env, &BTreeSet::new()); + assert!(out.contains("OLLAMA_MODEL=llama3.1"), "kept:\n{out}"); + assert!(out.contains("JWT_SECRET=\n"), "blanked:\n{out}"); + } + + #[test] + fn identity_reset_covers_host_keys_machine_id_and_cloud_init() { + let cmds = identity_reset_commands().join(" ; "); + assert!(cmds.contains("/etc/ssh/ssh_host_*"), "host keys: {cmds}"); + assert!(cmds.contains("/etc/machine-id"), "machine-id: {cmds}"); + assert!( + cmds.contains("/var/lib/cloud/instance"), + "cloud-init: {cmds}" + ); + } + + #[test] + fn volume_reset_preserves_the_kept_volumes() { + let cmds = volume_reset_commands("/home/trydirect/project", &["ollama"]).join(" ; "); + assert!( + cmds.contains("docker compose down"), + "stack stopped: {cmds}" + ); + assert!( + cmds.contains("grep -Ev '(ollama)'"), + "kept volume excluded from removal: {cmds}" + ); + } + + #[test] + fn volume_reset_without_a_keep_list_removes_everything() { + let cmds = volume_reset_commands("/home/trydirect/project", &[]).join(" ; "); + assert!(cmds.contains("| cat |"), "no filter applied: {cmds}"); + } + + #[test] + fn parse_env_pairs_skips_comments_blanks_and_empty_values() { + let env = "# header\n\nA=1\nEMPTY=\nB=two\n"; + let pairs = parse_env_pairs(env); + assert_eq!(pairs.get("A").map(String::as_str), Some("1")); + assert_eq!(pairs.get("B").map(String::as_str), Some("two")); + assert!(!pairs.contains_key("EMPTY"), "empty value is not a secret"); + } + + #[test] + fn protected_keys_cover_generated_and_provided_only() { + let contract = serde_json::json!({ + "services": { + "db": { "fields": { + "POSTGRES_PASSWORD": { "mutability": "generated" }, + "POSTGRES_USER": { "mutability": "fixed" } + }}, + "app": { "fields": { + "LICENSE_KEY": { "mutability": "provided" }, + "LOG_LEVEL": { "mutability": "editable" } + }} + } + }); + let keys = protected_keys_from_contract(&contract); + assert!(keys.contains("POSTGRES_PASSWORD")); + assert!(keys.contains("LICENSE_KEY")); + assert!(!keys.contains("POSTGRES_USER"), "fixed is not regenerated"); + assert!(!keys.contains("LOG_LEVEL"), "editable is not regenerated"); + } + + #[test] + fn protected_keys_from_a_shapeless_contract_is_empty() { + assert!(protected_keys_from_contract(&serde_json::Value::Null).is_empty()); + } + + #[test] + fn stackpilot_keeps_only_its_model_volume() { + assert_eq!(volumes_to_keep("stackpilot"), &["ollama"]); + // An unknown stack defaults to resetting everything — leaking is worse + // than rebuilding. + assert!(volumes_to_keep("something-else").is_empty()); + } +} diff --git a/src/helpers/mod.rs b/src/helpers/mod.rs index d741d19f..f071e56e 100644 --- a/src/helpers/mod.rs +++ b/src/helpers/mod.rs @@ -34,6 +34,7 @@ pub use dockerhub::*; pub use cloud::*; pub mod audit_cache; pub mod bake; +pub mod bake_finalize; pub mod bake_registry; pub mod cloud_init; pub mod compose_yaml; diff --git a/src/models/baked_snapshot.rs b/src/models/baked_snapshot.rs index 188b1e75..18f2a56a 100644 --- a/src/models/baked_snapshot.rs +++ b/src/models/baked_snapshot.rs @@ -19,6 +19,15 @@ pub struct BakedSnapshot { /// single value baked into the snapshot. `None` for snapshots baked before /// this column existed. pub config_contract: Option, + /// The `${VAR}` names the baked compose file references, captured at bake + /// time. The clone path checks the buyer's env against this before creating + /// a server: an unsatisfied reference resolves to an empty string at boot + /// with nothing but a Compose warning to show for it. + /// + /// `None` for snapshots baked before this column existed — those skip the + /// check rather than becoming undeployable. + #[sqlx(default)] + pub required_env_keys: Option, pub created_at: DateTime, } diff --git a/src/project_app/sync.rs b/src/project_app/sync.rs index 9fcf401c..f148393f 100644 --- a/src/project_app/sync.rs +++ b/src/project_app/sync.rs @@ -86,6 +86,7 @@ fn build_project_app( project_app.networks = app_networks_json(app, all_networks); project_app.enabled = Some(true); project_app.deploy_order = Some(deploy_order); + project_app.config_contract = app.config_contract.clone(); project_app } @@ -160,13 +161,24 @@ pub(crate) async fn sync_project_level_apps_from_form( .map(|app| (app.code.clone(), app.id)) .collect::>(); - for mut desired_app in desired_apps { + for desired_app in desired_apps { if let Some(existing_id) = existing_by_code.remove(&desired_app.code) { - desired_app.id = existing_id; - desired_app.deployment_id = None; - db::project_app::update(pool, &desired_app).await?; + let mut app = desired_app; + app.id = existing_id; + app.deployment_id = None; + db::project_app::update(pool, &app).await?; + if let Some(contract) = app.config_contract.clone() { + db::project_app::set_config_contract(pool, project_id, &app.code, contract) + .await?; + } } else { + let code = desired_app.code.clone(); + let contract = desired_app.config_contract.clone(); db::project_app::insert(pool, &desired_app).await?; + if let Some(contract) = contract { + db::project_app::set_config_contract(pool, project_id, &code, contract) + .await?; + } } } @@ -303,4 +315,76 @@ mod tests { assert!(apps.is_empty()); } + + #[test] + fn project_level_apps_from_form_propagates_config_contract() { + let form: ProjectForm = serde_json::from_value(json!({ + "custom": { + "custom_stack_code": "contract-project", + "project_name": "Contract project", + "networks": [ + {"id": "net-1", "name": "default_network"} + ], + "web": [{ + "_id": "web-1", + "name": "Website", + "code": "website", + "type": "web", + "custom": true, + "dockerhub_image": "nginx:1.27", + "domain": "example.com", + "restart": "always", + "network": ["net-1"], + "environment": [{"key": "JWT_SECRET", "value": "auto"}], + "shared_ports": [{"host_port": "80", "container_port": "8080"}], + "volumes": [], + "config_contract": { + "services": { + "web": { + "fields": { + "JWT_SECRET": { "mutability": "generated" } + } + } + } + } + }], + "service": [{ + "_id": "svc-1", + "name": "Redis", + "code": "redis", + "type": "service", + "custom": true, + "dockerhub_image": "redis:7-alpine", + "domain": "", + "restart": "unless-stopped", + "network": ["net-1"], + "environment": [], + "shared_ports": [], + "volumes": [] + }], + "feature": [] + } + })) + .expect("project form should deserialize"); + + let apps = project_level_apps_from_form(42, &form); + + assert_eq!( + apps[0].config_contract, + Some(json!({ + "services": { + "web": { + "fields": { + "JWT_SECRET": { "mutability": "generated" } + } + } + } + })), + "config_contract should be propagated from form app to project app" + ); + assert_eq!( + apps[1].config_contract, None, + "apps without config_contract should remain None" + ); + } } diff --git a/src/routes/agent/audit.rs b/src/routes/agent/audit.rs index 07dc5284..65cd6206 100644 --- a/src/routes/agent/audit.rs +++ b/src/routes/agent/audit.rs @@ -1,10 +1,10 @@ use crate::db::agent_audit_log as audit_db; -use crate::helpers::JsonResponse; +use crate::{helpers, models}; use crate::models::agent_audit_log::{AgentAuditLog, AuditBatchRequest}; -use actix_web::error::ErrorUnauthorized; -use actix_web::{get, post, web, HttpRequest, HttpResponse, Result}; +use actix_web::{get, post, web, HttpResponse, Result}; use serde::{Deserialize, Serialize}; use sqlx::PgPool; +use std::sync::Arc; // ── POST /api/v1/agent/audit ─────────────────────────────────────────────── @@ -13,20 +13,22 @@ pub struct IngestResponse { pub accepted: usize, } -/// Receive a batch of audit events from the Status Panel. +/// Receive a batch of audit events from the Status Panel agent. /// -/// Auth: `X-Internal-Key` header must match the `INTERNAL_SERVICES_ACCESS_KEY` -/// environment variable. +/// Auth: agent token via `X-Agent-Id` and `Bearer` headers (handled by +/// middleware). The installation hash must belong to the authenticated agent. #[tracing::instrument(name = "Agent audit ingest", skip_all)] #[post("/audit")] pub async fn agent_audit_ingest_handler( - req: HttpRequest, + agent: web::ReqData>, body: web::Json, pool: web::Data, ) -> Result { - // Shared with the other service-to-service endpoints; the comparison is - // now constant-time. - crate::helpers::internal_key::require_internal_key(&req)?; + if agent.deployment_hash != body.installation_hash { + return Err(helpers::JsonResponse::forbidden( + "Not authorized for this installation", + )); + } // Short-circuit on empty batch if body.events.is_empty() { @@ -36,7 +38,7 @@ pub async fn agent_audit_ingest_handler( let accepted = audit_db::insert_batch(&pool, &body.installation_hash, &body.events) .await .map_err(|err| { - JsonResponse::<()>::build() + helpers::JsonResponse::<()>::build() .internal_server_error(format!("Failed to store audit events: {}", err)) })?; @@ -72,7 +74,7 @@ pub async fn agent_audit_query_handler( // installation and prove they own it. let user = caller_user .as_deref() - .ok_or_else(|| JsonResponse::::forbidden("Authentication required"))?; + .ok_or_else(|| helpers::JsonResponse::::forbidden("Authentication required"))?; let is_admin = matches!(user.role.as_str(), "admin_service" | "group_admin" | "root"); @@ -80,7 +82,7 @@ pub async fn agent_audit_query_handler( let installation_hash = params .installation_hash .as_deref() - .ok_or_else(|| JsonResponse::::bad_request("installation_hash is required"))?; + .ok_or_else(|| helpers::JsonResponse::::bad_request("installation_hash is required"))?; crate::routes::agent::guard::authorize_deployment_access( &pool, @@ -100,7 +102,7 @@ pub async fn agent_audit_query_handler( ) .await .map_err(|err| { - JsonResponse::<()>::build() + helpers::JsonResponse::<()>::build() .internal_server_error(format!("Failed to fetch audit log: {}", err)) })?; diff --git a/src/routes/marketplace/creator.rs b/src/routes/marketplace/creator.rs index a4da4772..219105fd 100644 --- a/src/routes/marketplace/creator.rs +++ b/src/routes/marketplace/creator.rs @@ -165,25 +165,58 @@ pub async fn create_handler( let template = if let Some(existing_template) = existing { // Update existing template tracing::info!("Updating existing template with slug: {}", req.slug); - let updated = db::marketplace::update_metadata( - pg_pool.get_ref(), - &existing_template.id, - Some(&req.name), - req.short_description.as_deref(), - req.long_description.as_deref(), - req.category_code.as_deref(), - Some(tags.clone()), - Some(tech_stack.clone()), - Some(infrastructure_requirements.clone()), - Some(price), - Some(billing_cycle.as_str()), - req.required_plan_name.as_deref(), - Some(currency.as_str()), - req.public_ports.clone(), - req.vendor_url.as_deref(), - ) - .await - .map_err(|err| JsonResponse::::build().internal_server_error(err))?; + + // Use the resubmit-aware update for templates that are already + // submitted, under review, or approved — `update_metadata` only + // allows draft/rejected/needs_changes. + let updated = if matches!( + existing_template.status.as_str(), + "submitted" | "under_review" | "approved" + ) { + db::marketplace::update_metadata_for_resubmit( + pg_pool.get_ref(), + &existing_template.id, + Some(&req.name), + req.short_description.as_deref(), + req.long_description.as_deref(), + req.category_code.as_deref(), + Some(tags.clone()), + Some(tech_stack.clone()), + Some(infrastructure_requirements.clone()), + Some(price), + Some(billing_cycle.as_str()), + req.required_plan_name.as_deref(), + Some(currency.as_str()), + req.public_ports.clone(), + req.vendor_url.as_deref(), + ) + .await + .map_err(|err| { + JsonResponse::::build().internal_server_error(err) + })? + } else { + db::marketplace::update_metadata( + pg_pool.get_ref(), + &existing_template.id, + Some(&req.name), + req.short_description.as_deref(), + req.long_description.as_deref(), + req.category_code.as_deref(), + Some(tags.clone()), + Some(tech_stack.clone()), + Some(infrastructure_requirements.clone()), + Some(price), + Some(billing_cycle.as_str()), + req.required_plan_name.as_deref(), + Some(currency.as_str()), + req.public_ports.clone(), + req.vendor_url.as_deref(), + ) + .await + .map_err(|err| { + JsonResponse::::build().internal_server_error(err) + })? + }; if !updated { return Err(JsonResponse::::build() diff --git a/src/routes/oneclick_deploy/clone.rs b/src/routes/oneclick_deploy/clone.rs index 7f567f74..33b76a45 100644 --- a/src/routes/oneclick_deploy/clone.rs +++ b/src/routes/oneclick_deploy/clone.rs @@ -204,6 +204,44 @@ pub async fn clone_server( None => (Vec::new(), Vec::new()), }; + // ── fail closed on an env the image cannot boot with ───────────────── + // The bake pinned the `${VAR}` references its compose file actually needs. + // Compose resolves an unsatisfied reference to an *empty string* and only + // warns, and the unit's ExecStartPre ends in `|| true`, so a missing key + // produces a running-but-broken stack rather than a visible failure. Check + // before a server exists, so the buyer gets a refusal instead of a bill. + // + // Snapshots baked before `required_env_keys` carry NULL and skip this. + let provided: std::collections::BTreeSet = form + .env + .iter() + .filter(|(_, value)| !value.trim().is_empty()) + .map(|(key, _)| key.clone()) + .chain(regen.iter().map(|(key, _)| key.clone())) + .chain(regen_jwt.iter().map(|spec| spec.target_key.clone())) + .collect(); + + let missing = missing_required_env_keys(&snapshot.required_env_keys, &provided); + if !missing.is_empty() { + tracing::error!( + stack = %form.stack, + version = %snapshot.version, + missing = ?missing, + "refusing clone: baked compose references env keys the deploy would not supply" + ); + return HttpResponse::UnprocessableEntity().json(json!({ + "error": "Incomplete environment for this snapshot", + "details": format!( + "the baked image for '{}' v{} references {} environment variable(s) that this \ + deploy would not set ({}). They would resolve to empty strings at boot.", + form.stack, + snapshot.version, + missing.len(), + missing.join(", ") + ), + })); + } + // Render cloud-init with per-user env + domain. Secrets are pre-resolved (by // the user service) into `form.env`; `regen` mints fresh values for // `mutability: generated` fields on the box at first boot, reusing the same @@ -333,12 +371,11 @@ pub async fn clone_server( // template fallback stores a stack_definition blob that is not a // ProjectForm — parsing fails and the panel stays empty (the user can // add apps manually). - if let Ok(form) = serde_json::from_value::( - project.request_json.clone(), - ) { + if let Ok(form) = + serde_json::from_value::(project.request_json.clone()) + { if let Err(err) = - crate::project_app::sync_project_level_apps_from_form(&pg_pool, project.id, &form) - .await + crate::project_app::sync_project_level_apps_from_form(&pg_pool, project.id, &form).await { tracing::warn!( error = %err, @@ -652,6 +689,26 @@ pub async fn clone_server( }) } +/// The pinned `${VAR}` references the buyer's environment would leave unset. +/// +/// `required` is `baked_snapshots.required_env_keys` — a JSON array recorded at +/// bake time. `None` (a snapshot baked before that column) yields no misses, so +/// existing images keep deploying unchanged rather than becoming undeployable. +fn missing_required_env_keys( + required: &Option, + provided: &std::collections::BTreeSet, +) -> Vec { + let Some(serde_json::Value::Array(keys)) = required else { + return Vec::new(); + }; + + keys.iter() + .filter_map(serde_json::Value::as_str) + .filter(|key| !provided.contains(*key)) + .map(str::to_string) + .collect() +} + /// The generated fields whose fresh value must be minted on the cloned box, /// paired with the canonical shell generator for each. Reuses the *single* /// source of truth for the type→generator mapping @@ -744,13 +801,41 @@ fn derived_jwt_commands( #[cfg(test)] mod regen_tests { - use super::{derived_jwt_commands, regen_commands}; + use super::{derived_jwt_commands, missing_required_env_keys, regen_commands}; use serde_json::json; use std::collections::BTreeMap; /// A `generated` field with no installer-supplied value gets a regen command /// reusing the canonical shell generator; a `fixed` field and an /// already-supplied value are left alone. + #[test] + fn missing_required_env_keys_reports_only_unsatisfied_ones() { + let required = Some(json!(["ALPHA", "BETA", "GAMMA"])); + let provided: std::collections::BTreeSet = + ["ALPHA".to_string(), "GAMMA".to_string()] + .into_iter() + .collect(); + + assert_eq!( + missing_required_env_keys(&required, &provided), + vec!["BETA".to_string()] + ); + } + + /// Snapshots baked before the column must stay deployable. + #[test] + fn missing_required_env_keys_is_empty_for_legacy_snapshots() { + assert!(missing_required_env_keys(&None, &std::collections::BTreeSet::new()).is_empty()); + } + + #[test] + fn missing_required_env_keys_is_empty_when_all_supplied() { + let required = Some(json!(["POSTGRES_PASSWORD"])); + let provided: std::collections::BTreeSet = + ["POSTGRES_PASSWORD".to_string()].into_iter().collect(); + assert!(missing_required_env_keys(&required, &provided).is_empty()); + } + #[test] fn regen_commands_only_for_unset_generated_fields() { let contract: crate::cli::config_parser::ConfigContract = serde_json::from_value(json!({ diff --git a/src/routes/server/ssh_key.rs b/src/routes/server/ssh_key.rs index 0de36d81..4b8c4035 100644 --- a/src/routes/server/ssh_key.rs +++ b/src/routes/server/ssh_key.rs @@ -710,7 +710,7 @@ pub async fn validate_all( }; } - let vault_key_path = match &server.vault_key_path { + let _vault_key_path = match &server.vault_key_path { Some(p) if !p.is_empty() => p, _ => { return ValidateResponse { diff --git a/src/services/agent_sweeper.rs b/src/services/agent_sweeper.rs new file mode 100644 index 00000000..daa1c53e --- /dev/null +++ b/src/services/agent_sweeper.rs @@ -0,0 +1,76 @@ +//! Housekeeping for dead and malformed agent rows. +//! +//! The `agents` table accumulates rows that no longer serve a purpose: agents +//! whose deployment was deleted or soft-deleted, agents that never sent a +//! heartbeat, and rows whose `deployment_hash` is structurally invalid (e.g. a +//! raw agent token stored instead of a hash). This sweeper removes them on a +//! daily cadence. +//! +//! A row is considered "dead" only when **both** of the following hold: +//! +//! 1. The deployment is gone — either no matching row in `deployment`, or the +//! row exists with `deleted IS TRUE`. +//! 2. There is no sign of life within the retention window — neither a +//! `last_heartbeat` nor an `audit_log` entry (the latter protects agents +//! that are alive but failing authentication, since `auth_failure` is +//! recorded in `audit_log` while `last_heartbeat` only advances on +//! successful `wait`/`report`). +//! +//! Malformed rows (invalid `deployment_hash`) are deleted unconditionally, +//! without a retention window — such an agent can never authenticate by its own +//! hash. +//! +//! Agents whose deployment is alive but silent are **not** touched: the row is +//! the agent's identity, and removing it would require a full reinstall. + +use std::time::Duration; + +use sqlx::PgPool; + +use crate::db; + +/// How often to sweep. Rows appear rarely; daily is ample. +const TICK: Duration = Duration::from_secs(86_400); + +/// How long a dead agent row is kept before removal. Long enough that a +/// temporarily stopped server can come back without losing its identity. +const RETENTION_DAYS: i32 = 30; + +pub fn spawn(pg_pool: PgPool) { + tokio::spawn(async move { + tracing::info!( + "agent_sweeper started (tick={:?}, retention={} days)", + TICK, + RETENTION_DAYS + ); + loop { + // Sleep first: startup is busy enough, and nothing here is urgent. + tokio::time::sleep(TICK).await; + + // Malformed rows are independent of retention — log at warn because + // new appearances indicate a write path that still needs fixing. + match db::agent::sweep_malformed(&pg_pool).await { + Ok(0) => {} + Ok(count) => tracing::warn!( + "agent_sweeper: removed {} malformed agent row(s) — \ + the write path producing these has not been found yet", + count + ), + Err(err) => { + tracing::warn!("agent_sweeper: malformed sweep error: {}", err) + } + } + + match db::agent::sweep_dead(&pg_pool, RETENTION_DAYS).await { + Ok(0) => tracing::debug!("agent_sweeper: nothing to remove"), + Ok(count) => tracing::info!( + "agent_sweeper: removed {} dead agent row(s)", + count + ), + Err(err) => { + tracing::warn!("agent_sweeper: dead sweep error: {}", err) + } + } + } + }); +} diff --git a/src/services/mod.rs b/src/services/mod.rs index 7ecfb2d3..87912848 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -1,4 +1,5 @@ pub mod agent_dispatcher; +pub mod agent_sweeper; pub mod agent_token; pub mod config_renderer; pub mod dag_executor; diff --git a/src/startup.rs b/src/startup.rs index 9c44f0a1..ea66fc06 100644 --- a/src/startup.rs +++ b/src/startup.rs @@ -122,6 +122,11 @@ pub async fn run( // it touches nothing the dashboard shows, and skipping it costs a slowly // growing table rather than correctness. crate::services::deployment_container_sweeper::spawn(api_pool.get_ref().clone()); + // Removes dead agent rows (no live deployment, no recent activity) and + // rows with structurally invalid deployment_hash. Runs daily; skipping it + // means the agents table slowly accumulates noise that obscures real + // problems. + crate::services::agent_sweeper::spawn(api_pool.get_ref().clone()); let payout_provider = crate::services::init_payout_provider(&settings.payouts) .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err.to_string()))?; diff --git a/tests/agent_sweep.rs b/tests/agent_sweep.rs new file mode 100644 index 00000000..0f8388ad --- /dev/null +++ b/tests/agent_sweep.rs @@ -0,0 +1,265 @@ +mod common; + +use chrono::Utc; +use sqlx::Row; +use stacker::db; +use stacker::models::Agent; +use tokio::sync::OnceCell; + +static APP: OnceCell = OnceCell::const_new(); + +async fn app() -> common::TestAppWithVaultFresh { + common::get_or_init_vault_app_fresh(&APP) + .await + .expect("Failed to start test app") +} + +async fn create_project(pool: &sqlx::PgPool, user_id: &str) -> i32 { + sqlx::query_scalar::<_, i32>( + "INSERT INTO project (stack_id, user_id, name, metadata, created_at, updated_at) + VALUES (gen_random_uuid(), $1, $2, '{}'::jsonb, NOW(), NOW()) + RETURNING id", + ) + .bind(user_id) + .bind(format!("sweep-test-{}", uuid::Uuid::new_v4())) + .fetch_one(pool) + .await + .expect("Failed to create project") +} + +async fn create_deployment( + pool: &sqlx::PgPool, + project_id: i32, + user_id: &str, + deployment_hash: &str, + deleted: bool, +) { + sqlx::query( + "INSERT INTO deployment (project_id, deployment_hash, user_id, metadata, status, deleted, created_at, updated_at) + VALUES ($1, $2, $3, '{}'::jsonb, 'running', $4, NOW(), NOW())", + ) + .bind(project_id) + .bind(deployment_hash) + .bind(user_id) + .bind(deleted) + .execute(pool) + .await + .expect("Failed to create deployment"); +} + +async fn insert_agent(pool: &sqlx::PgPool, agent: &Agent) { + sqlx::query( + "INSERT INTO agents (id, deployment_hash, capabilities, version, system_info, + last_heartbeat, status, token_hash, token_hash_updated_at, + created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)", + ) + .bind(agent.id) + .bind(&agent.deployment_hash) + .bind(&agent.capabilities) + .bind(&agent.version) + .bind(&agent.system_info) + .bind(agent.last_heartbeat) + .bind(&agent.status) + .bind(&agent.token_hash) + .bind(agent.token_hash_updated_at) + .bind(agent.created_at) + .bind(agent.updated_at) + .execute(pool) + .await + .expect("Failed to insert agent"); +} + +async fn agent_exists(pool: &sqlx::PgPool, agent_id: uuid::Uuid) -> bool { + sqlx::query_scalar::<_, bool>("SELECT EXISTS(SELECT 1 FROM agents WHERE id = $1)") + .bind(agent_id) + .fetch_one(pool) + .await + .expect("Failed to check agent existence") +} + +async fn count_agents(pool: &sqlx::PgPool) -> i64 { + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM agents") + .fetch_one(pool) + .await + .expect("Failed to count agents") +} + +/// Case 1: deployment deleted, no activity → row deleted +#[tokio::test] +async fn sweep_dead_deletes_agent_with_deleted_deployment() { + let app = app().await; + let project_id = create_project(&app.db_pool, "test_user_id").await; + let hash = format!("deployment_{}", uuid::Uuid::new_v4()); + create_deployment(&app.db_pool, project_id, "test_user_id", &hash, true).await; + + let mut agent = Agent::new(hash); + agent.last_heartbeat = Some(Utc::now() - chrono::Duration::days(60)); + insert_agent(&app.db_pool, &agent).await; + + let removed = db::agent::sweep_dead(&app.db_pool, 30).await.unwrap(); + assert_eq!(removed, 1); + assert!(!agent_exists(&app.db_pool, agent.id).await); +} + +/// Case 2: deployment deleted, but fresh audit_log entry → row preserved +#[tokio::test] +async fn sweep_dead_preserves_agent_with_recent_audit_log() { + let app = app().await; + let project_id = create_project(&app.db_pool, "test_user_id").await; + let hash = format!("deployment_{}", uuid::Uuid::new_v4()); + create_deployment(&app.db_pool, project_id, "test_user_id", &hash, true).await; + + let mut agent = Agent::new(hash.clone()); + agent.last_heartbeat = Some(Utc::now() - chrono::Duration::days(60)); + insert_agent(&app.db_pool, &agent).await; + + // Insert a recent audit_log entry (auth_failure — agent is alive but failing auth) + sqlx::query( + "INSERT INTO audit_log (id, agent_id, deployment_hash, action, status, created_at) + VALUES ($1, $2, $3, 'auth_failure', 'failure', NOW())", + ) + .bind(uuid::Uuid::new_v4()) + .bind(agent.id) + .bind(&hash) + .execute(&app.db_pool) + .await + .expect("Failed to insert audit log"); + + let removed = db::agent::sweep_dead(&app.db_pool, 30).await.unwrap(); + assert_eq!(removed, 0); + assert!(agent_exists(&app.db_pool, agent.id).await); +} + +/// Case 3: deployment missing entirely, no activity → row deleted +#[tokio::test] +async fn sweep_dead_deletes_agent_with_missing_deployment() { + let app = app().await; + let hash = format!("deployment_{}", uuid::Uuid::new_v4()); + // No deployment row at all + + let mut agent = Agent::new(hash); + agent.last_heartbeat = Some(Utc::now() - chrono::Duration::days(60)); + insert_agent(&app.db_pool, &agent).await; + + let removed = db::agent::sweep_dead(&app.db_pool, 30).await.unwrap(); + assert_eq!(removed, 1); + assert!(!agent_exists(&app.db_pool, agent.id).await); +} + +/// Case 4: deployment alive, agent silent for a year → row preserved +#[tokio::test] +async fn sweep_dead_preserves_agent_with_live_deployment() { + let app = app().await; + let project_id = create_project(&app.db_pool, "test_user_id").await; + let hash = format!("deployment_{}", uuid::Uuid::new_v4()); + create_deployment(&app.db_pool, project_id, "test_user_id", &hash, false).await; + + let mut agent = Agent::new(hash); + agent.last_heartbeat = Some(Utc::now() - chrono::Duration::days(365)); + insert_agent(&app.db_pool, &agent).await; + + let removed = db::agent::sweep_dead(&app.db_pool, 30).await.unwrap(); + assert_eq!(removed, 0); + assert!(agent_exists(&app.db_pool, agent.id).await); +} + +/// Case 5: deployment_hash of length 86 (raw token) → row deleted +#[tokio::test] +async fn sweep_malformed_deletes_agent_with_86_char_hash() { + let app = app().await; + // 86-char base64url string — the pattern seen in production + let bad_hash = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwx\ + yz0123456789-_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef\ + ghij"; + + let agent = Agent::new(bad_hash.to_string()); + insert_agent(&app.db_pool, &agent).await; + + let removed = db::agent::sweep_malformed(&app.db_pool).await.unwrap(); + assert_eq!(removed, 1); + assert!(!agent_exists(&app.db_pool, agent.id).await); +} + +/// Case 6: empty deployment_hash → row deleted +#[tokio::test] +async fn sweep_malformed_deletes_agent_with_empty_hash() { + let app = app().await; + + let agent = Agent::new(String::new()); + insert_agent(&app.db_pool, &agent).await; + + let removed = db::agent::sweep_malformed(&app.db_pool).await.unwrap(); + assert_eq!(removed, 1); + assert!(!agent_exists(&app.db_pool, agent.id).await); +} + +/// Valid hashes are not affected by sweep_malformed +#[tokio::test] +async fn sweep_malformed_preserves_valid_hash() { + let app = app().await; + let project_id = create_project(&app.db_pool, "test_user_id").await; + let hash = format!("deployment_{}", uuid::Uuid::new_v4()); + create_deployment(&app.db_pool, project_id, "test_user_id", &hash, false).await; + + let agent = Agent::new(hash); + insert_agent(&app.db_pool, &agent).await; + + let removed = db::agent::sweep_malformed(&app.db_pool).await.unwrap(); + assert_eq!(removed, 0); + assert!(agent_exists(&app.db_pool, agent.id).await); +} + +/// Case 7: after agent deletion, audit_log records survive and retain deployment_hash +#[tokio::test] +async fn audit_log_preserved_after_agent_deletion() { + let app = app().await; + let project_id = create_project(&app.db_pool, "test_user_id").await; + let hash = format!("deployment_{}", uuid::Uuid::new_v4()); + create_deployment(&app.db_pool, project_id, "test_user_id", &hash, true).await; + + let mut agent = Agent::new(hash.clone()); + agent.last_heartbeat = Some(Utc::now() - chrono::Duration::days(60)); + insert_agent(&app.db_pool, &agent).await; + + let audit_id = uuid::Uuid::new_v4(); + sqlx::query( + "INSERT INTO audit_log (id, agent_id, deployment_hash, action, status, created_at) + VALUES ($1, $2, $3, 'register', 'success', NOW() - interval '60 days')", + ) + .bind(audit_id) + .bind(agent.id) + .bind(&hash) + .execute(&app.db_pool) + .await + .expect("Failed to insert audit log"); + + // sweep_dead should NOT delete this agent (audit_log blocks it via the + // retention window). But we want to test what happens when the agent IS + // deleted — so delete it directly. + db::agent::delete(&app.db_pool, agent.id).await.unwrap(); + + // Audit log should survive, with agent_id NULLed and deployment_hash preserved + let row = sqlx::query( + "SELECT agent_id, deployment_hash FROM audit_log WHERE id = $1", + ) + .bind(audit_id) + .fetch_one(&app.db_pool) + .await + .expect("Audit log row should still exist"); + + let agent_id: Option = row.get("agent_id"); + let dep_hash: Option = row.get("deployment_hash"); + assert!(agent_id.is_none(), "agent_id should be NULL after ON DELETE SET NULL"); + assert_eq!(dep_hash.as_deref(), Some(hash.as_str())); +} + +/// sweep_dead with no matching rows returns 0 +#[tokio::test] +async fn sweep_dead_returns_zero_when_nothing_to_remove() { + let app = app().await; + let before = count_agents(&app.db_pool).await; + let removed = db::agent::sweep_dead(&app.db_pool, 30).await.unwrap(); + assert_eq!(removed, 0); + assert_eq!(count_agents(&app.db_pool).await, before); +} diff --git a/tests/features/agent_audit.feature b/tests/features/agent_audit.feature index 985a8fdc..637c5570 100644 --- a/tests/features/agent_audit.feature +++ b/tests/features/agent_audit.feature @@ -2,15 +2,15 @@ Feature: Agent Audit Log As a service operator I can ingest and query audit events via the /api/v1/agent/audit endpoints. - Scenario: Ingest audit events with valid internal key + Scenario: Ingest audit events with a registered agent Given I am authenticated as User A When I ingest audit events for installation "bdd-audit-hash" Then the response status should be 200 - Scenario: Ingest audit events with invalid internal key + Scenario: Ingest audit events with an invalid agent token Given I am authenticated as User A - When I ingest audit events with invalid internal key - Then the response status should be 401 + When I ingest audit events with invalid agent token + Then the response status should be 400 # Querying is scoped to deployments the caller owns, so the installation # must belong to User A. Without that link the query is a cross-tenant read: diff --git a/tests/marketplace_create_template.rs b/tests/marketplace_create_template.rs index 164db9e3..e4b73b27 100644 --- a/tests/marketplace_create_template.rs +++ b/tests/marketplace_create_template.rs @@ -1665,3 +1665,161 @@ async fn submit_template_with_generated_policy_for_secret_field_is_accepted() { .expect("Failed to submit template for review"); assert_eq!(StatusCode::OK, submit_response.status()); } + +#[tokio::test] +async fn create_handler_updates_approved_template_metadata() { + let app = match common::spawn_app().await { + Some(app) => app, + None => return, + }; + let client = Client::new(); + + let create_response = create_template_with_body( + &client, + &app.address, + "test-bearer-token", + json!({ + "name": "Approved Metadata Template", + "slug": "approved-metadata-template", + "version": "1.0.0", + "stack_definition": { "services": { "web": { "image": "nginx:1.27" } } } + }), + ) + .await; + assert_eq!(StatusCode::CREATED, create_response.status()); + let template_id = create_response + .json::() + .await + .expect("Create response should be valid JSON")["item"]["id"] + .as_str() + .expect("Template id should be a string") + .to_string(); + + sqlx::query( + r#"UPDATE stack_template SET status = 'approved', approved_at = NOW() WHERE id = $1"#, + ) + .bind(Uuid::parse_str(&template_id).expect("Template id should be a UUID")) + .execute(&app.db_pool) + .await + .expect("Failed to mark template approved"); + + let update_response = create_template_with_body( + &client, + &app.address, + "test-bearer-token", + json!({ + "name": "Approved Metadata Template v2", + "slug": "approved-metadata-template", + "version": "1.0.0", + "stack_definition": { "services": { "web": { "image": "nginx:1.28" } } } + }), + ) + .await; + assert_eq!( + StatusCode::CREATED, + update_response.status(), + "POST /api/templates should succeed for approved templates (uses update_metadata_for_resubmit)" + ); + let body: Value = update_response + .json() + .await + .expect("Update response should be valid JSON"); + assert_eq!( + "Approved Metadata Template v2", + body["item"]["name"].as_str().expect("name should be a string"), + "Template name should be updated" + ); +} + +#[tokio::test] +async fn resubmit_approved_template_with_new_version_preserves_source_project_id() { + let _env_lock = env_lock().lock().expect("env lock should be available"); + let app = match common::spawn_app().await { + Some(app) => app, + None => return, + }; + let client = Client::new(); + + let project_id = common::create_test_project(&app.db_pool, "test_user_id").await; + common::create_test_deployment( + &app.db_pool, + "test_user_id", + project_id, + &format!("dpl-{}", Uuid::new_v4()), + ) + .await; + + let create_response = create_template_with_body( + &client, + &app.address, + "test-bearer-token", + json!({ + "name": "Resubmit Version Template", + "slug": "resubmit-version-template", + "source_project_id": project_id, + "version": "1.0.0", + "stack_definition": { "services": { "web": { "image": "nginx:1.27" } } } + }), + ) + .await; + assert_eq!(StatusCode::CREATED, create_response.status()); + let template_id = create_response + .json::() + .await + .expect("Create response should be valid JSON")["item"]["id"] + .as_str() + .expect("Template id should be a string") + .to_string(); + + sqlx::query( + r#"UPDATE stack_template SET status = 'approved', approved_at = NOW() WHERE id = $1"#, + ) + .bind(Uuid::parse_str(&template_id).expect("Template id should be a UUID")) + .execute(&app.db_pool) + .await + .expect("Failed to mark template approved"); + + let resubmit_response = client + .post(format!( + "{}/api/templates/{}/resubmit", + app.address, template_id + )) + .bearer_auth("test-bearer-token") + .json(&json!({ + "version": "1.1.0", + "stack_definition": { "services": { "web": { "image": "nginx:1.28" } } }, + "confirm_no_secrets": true, + "source_project_id": project_id + })) + .send() + .await + .expect("Failed to resubmit template"); + assert_eq!( + StatusCode::OK, + resubmit_response.status(), + "Resubmit with new version should succeed even when source_project_id was on the old version" + ); + + let stored_source_id: Option = sqlx::query_scalar( + r#"SELECT source_project_id FROM stack_template_version + WHERE template_id = $1::uuid AND is_latest = true"#, + ) + .bind(Uuid::parse_str(&template_id).expect("Template id should be a UUID")) + .fetch_one(&app.db_pool) + .await + .expect("Failed to query latest version source_project_id"); + assert_eq!( + Some(project_id), + stored_source_id, + "source_project_id should be persisted on the new version after resubmit" + ); + + let template_status: String = sqlx::query_scalar( + r#"SELECT status FROM stack_template WHERE id = $1::uuid"#, + ) + .bind(Uuid::parse_str(&template_id).expect("Template id should be a UUID")) + .fetch_one(&app.db_pool) + .await + .expect("Failed to query template status"); + assert_eq!("submitted", template_status, "Template should be in submitted status after resubmit"); +} diff --git a/tests/project_app_sync.rs b/tests/project_app_sync.rs index 67876680..20c5474b 100644 --- a/tests/project_app_sync.rs +++ b/tests/project_app_sync.rs @@ -223,3 +223,83 @@ async fn sync_project_updates_apps_without_creating_a_deployment() { .expect("deployments should load"); assert_eq!(deployment_count, 0, "sync must not create a deployment"); } + +#[tokio::test] +async fn sync_project_persists_config_contract_on_apps() { + let Some(app) = common::spawn_app().await else { + return; + }; + + let client = reqwest::Client::new(); + let create_response = client + .post(format!("{}/project", app.address)) + .header("Authorization", format!("Bearer {}", common::USER_A_TOKEN)) + .json(&json!({ + "custom": { + "custom_stack_code": "sync-contract-project", + "project_name": "Sync contract project", + "networks": [{ + "id": "default-network", + "name": "default_network" + }], + "web": [{ + "_id": "web-1", + "name": "Website", + "code": "website", + "type": "web", + "custom": true, + "dockerhub_image": "nginx:1.27", + "domain": "example.com", + "restart": "always", + "network": ["default-network"], + "environment": [{"key": "JWT_SECRET", "value": "auto"}], + "shared_ports": [{"host_port": "80", "container_port": "8080"}], + "volumes": [], + "config_contract": { + "services": { + "web": { + "fields": { + "JWT_SECRET": { "mutability": "generated" } + } + } + } + } + }], + "service": [], + "feature": [] + } + })) + .send() + .await + .expect("project create request should succeed"); + assert_eq!(create_response.status(), StatusCode::OK); + + let project_id = create_response + .json::() + .await + .expect("create response should be json")["item"]["id"] + .as_i64() + .expect("project id should be present") as i32; + + let apps = db::project_app::fetch_by_project(&app.db_pool, project_id) + .await + .expect("project apps should load"); + let website = apps + .iter() + .find(|app| app.code == "website") + .expect("website app should exist"); + + assert_eq!( + website.config_contract, + Some(json!({ + "services": { + "web": { + "fields": { + "JWT_SECRET": { "mutability": "generated" } + } + } + } + })), + "config_contract should be persisted on the project app during sync" + ); +} diff --git a/tests/steps/agent.rs b/tests/steps/agent.rs index 306bf771..fc30bc9f 100644 --- a/tests/steps/agent.rs +++ b/tests/steps/agent.rs @@ -328,12 +328,59 @@ async fn enqueue_command(world: &mut StepWorld, deployment_hash: String, cmd_typ // ─── Audit steps ───────────────────────────────────────────────── +async fn register_audit_agent(world: &StepWorld, installation_hash: &str) -> (String, String) { + let response = world + .client + .post(format!("{}/api/v1/agent/register", world.base_url)) + .header("X-Internal-Key", crate::steps::common::BDD_INTERNAL_KEY) + .json(&json!({ + "deployment_hash": installation_hash, + "agent_version": "1.0.0-bdd", + "capabilities": ["audit"], + "system_info": { "os": "linux", "arch": "x86_64" } + })) + .send() + .await + .expect("Audit agent registration request failed"); + + let status = response.status(); + let body: serde_json::Value = response + .json() + .await + .expect("Audit agent registration response should be JSON"); + assert!( + status.is_success(), + "Audit agent registration failed: {status} {body}" + ); + + let item = body + .pointer("/data/item") + .or_else(|| body.pointer("/item")) + .expect("Audit agent registration response should contain an item"); + let agent_id = item["agent_id"] + .as_str() + .expect("Audit agent registration should return agent_id") + .to_string(); + let agent_token = item["agent_token"] + .as_str() + .expect("Audit agent registration should return agent_token") + .to_string(); + + (agent_id, agent_token) +} + async fn do_ingest( world: &mut StepWorld, installation_hash: &str, - key: &str, + token: &str, events: serde_json::Value, ) { + let (agent_id, agent_token) = register_audit_agent(world, installation_hash).await; + let auth_token = if token == "wrong-key" { + token.to_string() + } else { + agent_token + }; let url = format!("{}/api/v1/agent/audit", world.base_url); let body = json!({ "installation_hash": installation_hash, @@ -342,8 +389,8 @@ async fn do_ingest( let resp = world .client .post(&url) - .header("Authorization", format!("Bearer {}", world.auth_token)) - .header("x-internal-key", key) + .header("X-Agent-Id", agent_id) + .header("Authorization", format!("Bearer {}", auth_token)) .json(&body) .send() .await @@ -364,7 +411,7 @@ async fn ingest_audit(world: &mut StepWorld, installation_hash: String) { do_ingest(world, &installation_hash, "bdd-internal-key", events).await; } -#[when("I ingest audit events with invalid internal key")] +#[when("I ingest audit events with invalid agent token")] async fn ingest_audit_invalid_key(world: &mut StepWorld) { let events = json!([ {"id": 1, "event_type": "test", "payload": {}, "created_at": 1711000000}