diff --git a/migrations/20260919120000_baked_snapshots_required_env_keys.down.sql b/migrations/20260919120000_baked_snapshots_required_env_keys.down.sql new file mode 100644 index 00000000..1f427e62 --- /dev/null +++ b/migrations/20260919120000_baked_snapshots_required_env_keys.down.sql @@ -0,0 +1 @@ +ALTER TABLE baked_snapshots DROP COLUMN required_env_keys; diff --git a/migrations/20260919120000_baked_snapshots_required_env_keys.up.sql b/migrations/20260919120000_baked_snapshots_required_env_keys.up.sql new file mode 100644 index 00000000..3a35e504 --- /dev/null +++ b/migrations/20260919120000_baked_snapshots_required_env_keys.up.sql @@ -0,0 +1,10 @@ +-- Record which ${VAR} references the baked compose actually needs, so the clone +-- path can refuse a deploy whose env file would not satisfy them. +-- +-- Without this, a missing key is silent: Docker Compose substitutes an empty +-- string with only a warning, the unit's ExecStartPre ends in `|| true`, and the +-- systemd unit still reports active while the stack is misconfigured. +-- +-- Nullable on purpose: snapshots baked before this column carry NULL and skip +-- the check, so existing images keep deploying unchanged. +ALTER TABLE baked_snapshots ADD COLUMN required_env_keys JSONB; diff --git a/src/bin/bake.rs b/src/bin/bake.rs index fb5ba0fb..547d5a96 100644 --- a/src/bin/bake.rs +++ b/src/bin/bake.rs @@ -7,7 +7,13 @@ //! Usage: //! HETZNER_TOKEN=... DATABASE_URL=... cargo run --bin bake -- \ //! --ip --stack ai-workflows-v2 --version 1.0.0 \ -//! --health-url http:///health +//! --health-url http:///health --ssh-key ~/.ssh/id_ed25519 +//! +//! `--ssh-key` is required: before snapshotting we sanitize the build box +//! (strip machine identity, blank the author's `.env`, parameterize secrets +//! embedded in compose values, drop initialized data volumes). Without it the +//! image would carry the author's credentials to every buyer, so the bake is +//! refused unless `--allow-unsanitized-snapshot` is passed deliberately. //! //! `DATABASE_URL` (the stacker Postgres) persists the BakeRecord; without it //! the bake still snapshots and prints the record, but it is not registered. @@ -23,6 +29,10 @@ async fn main() -> Result<(), Box> { let mut stack = "lamp".to_string(); let mut version = "v1".to_string(); let mut health_url: Option = None; + let mut ssh_key: Option = None; + let mut ssh_user = "root".to_string(); + let mut project_dir = "/home/trydirect/project".to_string(); + let mut allow_unsanitized = false; let mut i = 1; while i < args.len() { @@ -47,6 +57,22 @@ async fn main() -> Result<(), Box> { health_url = args.get(i + 1).cloned(); i += 2; } + "--ssh-key" => { + ssh_key = args.get(i + 1).cloned(); + i += 2; + } + "--ssh-user" => { + ssh_user = args.get(i + 1).cloned().unwrap_or(ssh_user); + i += 2; + } + "--project-dir" => { + project_dir = args.get(i + 1).cloned().unwrap_or(project_dir); + i += 2; + } + "--allow-unsanitized-snapshot" => { + allow_unsanitized = true; + i += 1; + } other => { eprintln!("ignoring unknown arg: {other}"); i += 1; @@ -77,7 +103,77 @@ async fn main() -> Result<(), Box> { let target = HetznerSnapshotTarget { provider_server_id: server_id, server_name: None, - public_ip: ip, + public_ip: ip.clone(), + }; + + // Resolve the author's field policy *before* finalizing: it decides which + // keys get blanked in `.env` and which names an embedded secret may be + // parameterized to. Also pinned to the snapshot further down so the clone + // path can regenerate those fields per buyer. + let pool = match std::env::var("DATABASE_URL") { + Ok(db_url) => Some(sqlx::PgPool::connect(&db_url).await?), + Err(_) => { + eprintln!("WARNING: DATABASE_URL not set — bake will NOT be registered in the snapshot registry."); + None + } + }; + + let config_contract = match &pool { + Some(pool) => resolve_config_contract(pool, &stack).await, + None => None, + }; + let protected_keys = config_contract + .as_ref() + .map(stacker::helpers::bake_finalize::protected_keys_from_contract) + .unwrap_or_default(); + + // Sanitize the build box before the snapshot is taken. + let finalize_outcome = match (&ssh_key, allow_unsanitized) { + (Some(key_path), _) => { + let Some(host) = ip.clone() else { + return Err("--ssh-key needs --ip (the build box address to connect to)".into()); + }; + let private_key_pem = std::fs::read_to_string(key_path) + .map_err(|e| format!("could not read --ssh-key {key_path}: {e}"))?; + + eprintln!("==> Finalizing build box before snapshot..."); + let ctx = stacker::helpers::bake_finalize::FinalizeContext { + host, + port: 22, + user: ssh_user.clone(), + private_key_pem, + project_dir: project_dir.clone(), + stack: stack.clone(), + protected_keys: protected_keys.clone(), + }; + let outcome = stacker::helpers::bake_finalize::finalize_build_box(&ctx).await?; + eprintln!( + " Sanitized. Compose requires {} env key(s): {}", + outcome.required_env_keys.len(), + outcome + .required_env_keys + .iter() + .cloned() + .collect::>() + .join(", ") + ); + Some(outcome) + } + (None, true) => { + eprintln!( + "WARNING: --allow-unsanitized-snapshot set. The image will keep the author's \ + .env values, data volumes and SSH host keys. Do NOT publish it to buyers." + ); + None + } + (None, false) => { + return Err( + "--ssh-key is required so the build box can be sanitized before \ + snapshotting (pass --allow-unsanitized-snapshot to skip, for a \ + private image only)" + .into(), + ) + } }; let connector = HetznerCloudClient::from_env().map_err(|e| e.to_string())?; @@ -94,56 +190,16 @@ async fn main() -> Result<(), Box> { ); // Persist into the snapshot registry so /api/v1/deploy/clone can resolve it. - if let Ok(db_url) = std::env::var("DATABASE_URL") { - let pool = sqlx::PgPool::connect(&db_url).await?; - - // Pin the author's field policy to this image so the clone path can - // regenerate `mutability: generated` fields fresh per buyer instead of - // shipping the single value baked into the snapshot. Resolved by the same - // slug the registry keys on (record.stack == stack_template.slug). - // Best-effort: an un-catalogued or unapproved stack bakes with no - // contract, and the clone path degrades to the baked values. - let config_contract = - match stacker::db::marketplace::get_approved_by_slug(&pool, &record.stack).await { - Ok(Some(template)) => { - eprintln!( - "DEBUG: resolved template '{}' id={} for stack '{}'", - template.name, template.id, record.stack - ); - match stacker::db::marketplace::get_config_contract(&pool, template.id).await { - Ok(serde_json::Value::Null) => { - eprintln!("DEBUG: config_contract is Null for template id={}", template.id); - None - } - Ok(contract) => { - eprintln!("DEBUG: config_contract resolved, keys={:?}", - contract.as_object().map(|o| o.keys().collect::>())); - Some(contract) - } - Err(err) => { - eprintln!( - "WARNING: could not read config_contract for '{}': {err}", - record.stack - ); - None - } - } - } - Ok(None) => { - eprintln!("DEBUG: no approved template found for stack '{}'", record.stack); - None - } - Err(err) => { - eprintln!( - "WARNING: could not resolve template for '{}': {err}", - record.stack - ); - None - } - }; - - eprintln!("DEBUG: config_contract to record: {:?}", - config_contract.as_ref().map(|c| c.as_object().map(|o| o.keys().collect::>()))); + if let Some(pool) = pool { + let required_env_keys = finalize_outcome.as_ref().map(|outcome| { + serde_json::Value::Array( + outcome + .required_env_keys + .iter() + .map(|key| serde_json::Value::String(key.clone())) + .collect(), + ) + }); let row = stacker::db::baked_snapshot::record( &pool, @@ -154,6 +210,7 @@ async fn main() -> Result<(), Box> { record.healthy, None, config_contract, + required_env_keys, ) .await .map_err(|e| e.to_string())?; @@ -161,9 +218,38 @@ async fn main() -> Result<(), Box> { "Registered snapshot in registry: id={} stack={}:{} image_id={}", row.id, row.stack, row.version, row.image_id ); - } else { - eprintln!("WARNING: DATABASE_URL not set — bake NOT registered in the snapshot registry."); } Ok(()) } + +/// The author's field policy for `stack`, resolved by the same slug the +/// snapshot registry keys on (`baked_snapshots.stack == stack_template.slug`). +/// +/// Best-effort: an un-catalogued or unapproved stack bakes with no contract and +/// the clone path degrades to the baked values. +async fn resolve_config_contract(pool: &sqlx::PgPool, stack: &str) -> Option { + match stacker::db::marketplace::get_approved_by_slug(pool, stack).await { + Ok(Some(template)) => { + match stacker::db::marketplace::get_config_contract(pool, template.id).await { + Ok(serde_json::Value::Null) => { + eprintln!("WARNING: template '{stack}' declares no config_contract — nothing will be regenerated per buyer."); + None + } + Ok(contract) => Some(contract), + Err(err) => { + eprintln!("WARNING: could not read config_contract for '{stack}': {err}"); + None + } + } + } + Ok(None) => { + eprintln!("WARNING: no approved template found for stack '{stack}'."); + None + } + Err(err) => { + eprintln!("WARNING: could not resolve template for '{stack}': {err}"); + None + } + } +} diff --git a/src/cli/generator/compose.rs b/src/cli/generator/compose.rs index 41780286..75eeb85f 100644 --- a/src/cli/generator/compose.rs +++ b/src/cli/generator/compose.rs @@ -842,12 +842,221 @@ pub fn parameterize_compose_env_vars( result } +/// Shortest value we will treat as a secret when searching *inside* other +/// values. Short strings ("admin", "postgres", a port) collide with ordinary +/// text and would corrupt the compose file. +const MIN_EMBEDDED_SECRET_LEN: usize = 12; + +/// Two protected fields resolved to the same literal value, so the compose +/// cannot be parameterized unambiguously. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EmbeddedSecretConflict { + /// The contract field names that share one value, sorted. + pub keys: Vec, +} + +impl std::fmt::Display for EmbeddedSecretConflict { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "the same secret value is declared under {} protected fields ({}). \ + Each is regenerated independently per buyer, so they would receive \ + different values and the stack would fail to authenticate. Declare \ + one field and reference it from the others.", + self.keys.len(), + self.keys.join(", ") + ) + } +} + +/// Replace secret values that appear *inside* a larger value with a `${KEY}` +/// reference — the case [`parameterize_compose_env_vars`] structurally cannot +/// reach, because it matches whole values by key name. +/// +/// The motivating case is a DSN: `DATABASE_URL: +/// postgresql://user:@host/db` carries the database password inside +/// its value, under a key name (`DATABASE_URL`) that no secret-name heuristic +/// recognises. Every existing protection layer keys off the *variable name*, so +/// such a credential is invisible to all of them at once and stays literal in +/// the baked image. +/// +/// `env_values` are the build box's resolved `KEY=value` pairs; `protected` are +/// the contract fields with `mutability: generated`/`provided`, i.e. the ones +/// that will be regenerated on the buyer's box and therefore can be referenced +/// safely. +pub fn parameterize_embedded_secret_values( + compose_content: &str, + env_values: &std::collections::BTreeMap, + protected: &std::collections::BTreeSet, +) -> Result { + // Every place a literal value is known by a name: the build box's .env plus + // the compose's own `KEY: value` pairs (a value interpolated into a service + // env block has no .env entry of its own). + let mut names_by_value: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + + for (key, value) in env_values + .iter() + .chain(compose_literal_env_pairs(compose_content).iter()) + { + if value.len() < MIN_EMBEDDED_SECRET_LEN { + continue; + } + names_by_value + .entry(value.clone()) + .or_default() + .insert(key.clone()); + } + + // A value reachable under two *protected* names diverges at regeneration + // time — refuse rather than bake an image that cannot boot. + let mut substitutions: Vec<(String, String)> = Vec::new(); + for (value, names) in &names_by_value { + let protected_names: Vec = names + .iter() + .filter(|name| protected.contains(*name)) + .cloned() + .collect(); + + match protected_names.len() { + 0 => continue, // not a managed secret; nothing regenerates it + 1 => substitutions.push((value.clone(), protected_names[0].clone())), + _ => { + return Err(EmbeddedSecretConflict { + keys: protected_names, + }) + } + } + } + + // Longest first, so a value that contains another is replaced whole. + substitutions.sort_by(|a, b| b.0.len().cmp(&a.0.len()).then(a.0.cmp(&b.0))); + + Ok(rewrite_environment_lines(compose_content, |line| { + let mut rewritten = line.to_string(); + for (value, key) in &substitutions { + if rewritten.contains(value.as_str()) { + rewritten = rewritten.replace(value.as_str(), &format!("${{{key}}}")); + } + } + rewritten + })) +} + +/// The literal `KEY: value` pairs declared in service `environment:` blocks. +fn compose_literal_env_pairs(compose_content: &str) -> std::collections::BTreeMap { + let mut pairs = std::collections::BTreeMap::new(); + + for_each_environment_line(compose_content, |line| { + if let Some((key, value)) = line.trim().split_once(':') { + let key = key.trim(); + let value = value.trim().trim_matches('"').trim_matches('\''); + if is_env_identifier(key) && !value.is_empty() && !value.starts_with("${") { + pairs.insert(key.to_string(), value.to_string()); + } + } + }); + + pairs +} + +/// Apply `rewrite` to every line inside a service `environment:` block, +/// leaving the rest of the document untouched. +fn rewrite_environment_lines( + compose_content: &str, + mut rewrite: impl FnMut(&str) -> String, +) -> String { + let mut result = String::with_capacity(compose_content.len()); + + for (line, in_environment) in environment_lines(compose_content) { + if in_environment { + result.push_str(&rewrite(line)); + } else { + result.push_str(line); + } + result.push('\n'); + } + + result +} + +fn for_each_environment_line(compose_content: &str, mut visit: impl FnMut(&str)) { + for (line, in_environment) in environment_lines(compose_content) { + if in_environment { + visit(line); + } + } +} + +/// Pair every line with whether it sits inside a service `environment:` block. +/// +/// Shares the block-tracking rule used by [`parameterize_compose_env_vars`]: +/// the block ends at the first non-blank line indented at or above the +/// `environment:` key itself. +fn environment_lines(compose_content: &str) -> Vec<(&str, bool)> { + let mut in_environment = false; + let mut env_indent = 0usize; + let mut out = Vec::new(); + + for line in compose_content.lines() { + let trimmed = line.trim_start(); + let indent = line.len() - trimmed.len(); + + if trimmed == "environment:" { + in_environment = true; + env_indent = indent; + out.push((line, false)); + continue; + } + + if in_environment && (trimmed.is_empty() || indent <= env_indent) { + in_environment = false; + } + + out.push((line, in_environment)); + } + + out +} + +/// The `${VAR}` names a compose file references, in sorted order. +/// +/// Captured at bake time and pinned to the snapshot so the clone path can +/// verify the buyer's environment satisfies the image *before* a server is +/// created. Compose resolves an unsatisfied reference to an empty string and +/// only warns, so without this check the failure surfaces as a misconfigured +/// stack rather than a refused deploy. +pub fn collect_env_var_references(compose_content: &str) -> std::collections::BTreeSet { + let mut names = std::collections::BTreeSet::new(); + let bytes = compose_content.as_bytes(); + let mut i = 0usize; + + while i + 1 < bytes.len() { + if bytes[i] != b'$' || bytes[i + 1] != b'{' { + i += 1; + continue; + } + let Some(end) = compose_content[i + 2..].find('}') else { + break; + }; + let raw = &compose_content[i + 2..i + 2 + end]; + // Compose allows ${VAR:-default} / ${VAR-default} / ${VAR:?err}. + let name = raw.split([':', '-', '?', '+']).next().unwrap_or("").trim(); + if is_env_identifier(name) { + names.insert(name.to_string()); + } + i += 2 + end + 1; + } + + names +} + /// Returns `true` when `s` looks like a POSIX env-variable name. fn is_env_identifier(s: &str) -> bool { !s.is_empty() - && s.chars().enumerate().all(|(i, c)| { - c.is_ascii_alphanumeric() || c == '_' || (i == 0 && c == '.') - }) + && s.chars() + .enumerate() + .all(|(i, c)| c.is_ascii_alphanumeric() || c == '_' || (i == 0 && c == '.')) } // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ @@ -1962,12 +2171,27 @@ services: let result = parameterize_compose_env_vars(compose, &keys); - assert!(result.contains("ADMIN_PASSWORD: ${ADMIN_PASSWORD}"), "admin pw:\n{result}"); - assert!(result.contains("SECRET_KEY: ${SECRET_KEY}"), "secret key:\n{result}"); - assert!(result.contains("DATABASE_URL: ${DATABASE_URL}"), "db url:\n{result}"); + assert!( + result.contains("ADMIN_PASSWORD: ${ADMIN_PASSWORD}"), + "admin pw:\n{result}" + ); + assert!( + result.contains("SECRET_KEY: ${SECRET_KEY}"), + "secret key:\n{result}" + ); + assert!( + result.contains("DATABASE_URL: ${DATABASE_URL}"), + "db url:\n{result}" + ); // Non-secret values stay as-is. - assert!(result.contains("ADMIN_USER: admin"), "admin user:\n{result}"); - assert!(result.contains("OLLAMA_MODEL: llama3.1"), "model:\n{result}"); + assert!( + result.contains("ADMIN_USER: admin"), + "admin user:\n{result}" + ); + assert!( + result.contains("OLLAMA_MODEL: llama3.1"), + "model:\n{result}" + ); } #[test] @@ -1988,11 +2212,153 @@ services: let result = parameterize_compose_env_vars(compose, &keys); - assert!(result.contains("SECRET_KEY: ${SECRET_KEY}"), "secret:\n{result}"); - assert!(result.contains("my.stacker.service: myapp"), "label:\n{result}"); + assert!( + result.contains("SECRET_KEY: ${SECRET_KEY}"), + "secret:\n{result}" + ); + assert!( + result.contains("my.stacker.service: myapp"), + "label:\n{result}" + ); assert!(result.contains("- app_data:/app/data"), "volume:\n{result}"); } + // ── embedded secret values (secrets inside a larger value) ────────────── + + fn env_map(pairs: &[(&str, &str)]) -> std::collections::BTreeMap { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + fn key_set(keys: &[&str]) -> std::collections::BTreeSet { + keys.iter().map(|k| k.to_string()).collect() + } + + #[test] + fn embedded_secret_inside_a_dsn_is_parameterized() { + // The case whole-value replacement structurally cannot reach. + let compose = "\ +services: + app: + environment: + DATABASE_URL: postgresql://stackpilot:2213a996143863b99a0f2d3e22907690@db:5432/stackpilot +"; + let env = env_map(&[("POSTGRES_PASSWORD", "2213a996143863b99a0f2d3e22907690")]); + let result = + parameterize_embedded_secret_values(compose, &env, &key_set(&["POSTGRES_PASSWORD"])) + .expect("no conflict"); + + assert!( + result.contains("postgresql://stackpilot:${POSTGRES_PASSWORD}@db:5432/stackpilot"), + "password replaced in place:\n{result}" + ); + assert!( + !result.contains("2213a996143863b99a0f2d3e22907690"), + "no literal left:\n{result}" + ); + } + + #[test] + fn short_values_are_left_alone() { + // "admin" would otherwise corrupt every word containing it. + let compose = "services:\n app:\n environment:\n ADMIN_USER: admin\n GREETING: administrator\n"; + let env = env_map(&[("ADMIN_USER", "admin")]); + let result = parameterize_embedded_secret_values(compose, &env, &key_set(&["ADMIN_USER"])) + .expect("no conflict"); + assert!( + result.contains("GREETING: administrator"), + "untouched:\n{result}" + ); + } + + #[test] + fn undeclared_values_are_left_alone() { + // Nothing regenerates it on the buyer's box, so a ${REF} would resolve empty. + let compose = + "services:\n app:\n environment:\n URL: http://host/aaaaaaaaaaaaaaaa\n"; + let env = env_map(&[("SOME_KEY", "aaaaaaaaaaaaaaaa")]); + let result = + parameterize_embedded_secret_values(compose, &env, &std::collections::BTreeSet::new()) + .expect("no conflict"); + assert!( + result.contains("aaaaaaaaaaaaaaaa"), + "left literal:\n{result}" + ); + } + + #[test] + fn one_value_under_two_protected_names_is_refused() { + // stackpilot's DB_PASSWORD/POSTGRES_PASSWORD duplication: both are + // `generated`, so regeneration would hand them different values. + let compose = "\ +services: + db: + environment: + POSTGRES_PASSWORD: 2213a996143863b99a0f2d3e22907690 +"; + let env = env_map(&[("DB_PASSWORD", "2213a996143863b99a0f2d3e22907690")]); + let err = parameterize_embedded_secret_values( + compose, + &env, + &key_set(&["DB_PASSWORD", "POSTGRES_PASSWORD"]), + ) + .expect_err("duplicate must be refused"); + + assert_eq!( + err.keys, + vec!["DB_PASSWORD".to_string(), "POSTGRES_PASSWORD".to_string()] + ); + } + + #[test] + fn non_environment_blocks_are_never_rewritten() { + let compose = "\ +services: + app: + image: myapp:2213a996143863b99a0f2d3e22907690 + environment: + TOKEN: 2213a996143863b99a0f2d3e22907690 +"; + let env = env_map(&[("TOKEN", "2213a996143863b99a0f2d3e22907690")]); + let result = parameterize_embedded_secret_values(compose, &env, &key_set(&["TOKEN"])) + .expect("no conflict"); + + assert!( + result.contains("image: myapp:2213a996143863b99a0f2d3e22907690"), + "image digest untouched:\n{result}" + ); + assert!( + result.contains("TOKEN: ${TOKEN}"), + "env replaced:\n{result}" + ); + } + + #[test] + fn collects_env_var_references_with_defaults_and_ignores_literals() { + let compose = "\ +services: + app: + image: app:latest + environment: + A: ${ALPHA} + B: ${BETA:-fallback} + C: ${GAMMA?required} + D: plain-value +"; + let refs = collect_env_var_references(compose); + let found: Vec<&str> = refs.iter().map(String::as_str).collect(); + assert_eq!(found, vec!["ALPHA", "BETA", "GAMMA"]); + } + + #[test] + fn collects_env_var_reference_embedded_in_a_dsn() { + let compose = + "services:\n app:\n environment:\n DATABASE_URL: postgres://u:${PW}@h/db\n"; + assert!(collect_env_var_references(compose).contains("PW")); + } + #[test] fn parameterize_no_keys_returns_original() { let compose = "services:\n app:\n environment:\n FOO: bar\n"; diff --git a/src/db/baked_snapshot.rs b/src/db/baked_snapshot.rs index 7aed2106..160839f2 100644 --- a/src/db/baked_snapshot.rs +++ b/src/db/baked_snapshot.rs @@ -64,11 +64,12 @@ pub async fn record( healthy: bool, digests: Option, config_contract: Option, + required_env_keys: Option, ) -> Result { sqlx::query_as::<_, BakedSnapshot>( r#" - INSERT INTO baked_snapshots (stack, version, provider, image_id, healthy, digests, config_contract) - VALUES ($1, $2, $3, $4, $5, $6, $7) + INSERT INTO baked_snapshots (stack, version, provider, image_id, healthy, digests, config_contract, required_env_keys) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING * "#, ) @@ -79,6 +80,7 @@ pub async fn record( .bind(healthy) .bind(digests) .bind(config_contract) + .bind(required_env_keys) .fetch_one(pool) .await .map_err(|e| format!("Failed to record baked snapshot: {e}")) diff --git a/src/helpers/bake.rs b/src/helpers/bake.rs index 658446a1..7503890e 100644 --- a/src/helpers/bake.rs +++ b/src/helpers/bake.rs @@ -35,6 +35,11 @@ pub enum BakeError { NoImageId, #[error("bake backend error: {0}")] Backend(String), + /// The build box could not be sanitized before snapshotting. Refused rather + /// than published: an unsanitized image carries the author's credentials to + /// every buyer. + #[error("bake rejected: could not finalize build box: {0}")] + Finalize(String), } /// Pure gate: only a healthy build box with a real `image_id` yields a record. diff --git a/src/helpers/bake_finalize.rs b/src/helpers/bake_finalize.rs new file mode 100644 index 00000000..56eb5c89 --- /dev/null +++ b/src/helpers/bake_finalize.rs @@ -0,0 +1,434 @@ +//! Finalize a build box before it is snapshotted (immutable-deploy BAKE step). +//! +//! A bake snapshots the *whole disk* of a build box that the author deployed +//! normally. That disk carries three classes of the author's own secrets, none +//! of which a buyer's clone should inherit: +//! +//! 1. **Machine identity** — SSH host keys and `machine-id`. Left in place, +//! every clone of the image shares them, so any buyer can impersonate the +//! SSH host of any other buyer. This is the standard "sysprep" step every +//! golden-image pipeline performs (`virt-sysprep`, Packer, the DigitalOcean +//! 1-Click checklist); we had none. +//! 2. **The co-located `.env`** — shipped verbatim next to the compose file by +//! the deploy bundle, so the author's literal secrets sit in the image even +//! when the compose itself is fully parameterized. Parameterizing compose +//! alone just moves the secret from one file in the image to another. +//! 3. **Initialized data volumes** — a secret the app wrote into its own +//! database/volume on first run is frozen in the snapshot and is *not* +//! governed by environment variables any more. Postgres is the canonical +//! case: `POSTGRES_PASSWORD` is honoured only when it initializes an empty +//! data directory, so a cloned box silently keeps the author's role password. +//! +//! Everything here is a pure function producing shell, so the policy is +//! unit-tested without infra; [`crate::helpers::bake`] wires it to a real SSH +//! session. + +use std::collections::BTreeSet; + +/// Volumes whose content must survive the bake, keyed by stack slug. +/// +/// Defaulting to *reset* is deliberate: wrongly resetting a volume costs a +/// rebuild of cheap state, while wrongly keeping one leaks the author's +/// credentials to every buyer. Only volumes that are expensive to rebuild +/// **and** carry no credentials belong here. +/// +/// `stackpilot`'s Ollama volume holds the pulled model weights — gigabytes, +/// with a 600s pull timeout in `scripts/download-model.sh`. Preserving it is +/// the entire economic point of baking that stack. +pub fn volumes_to_keep(stack: &str) -> &'static [&'static str] { + match stack { + "stackpilot" => &["ollama"], + _ => &[], + } +} + +/// Shell to strip machine identity so each clone boots as a distinct host. +/// +/// `sshd` regenerates host keys on first boot when none are present, and +/// `systemd` repopulates an empty `/etc/machine-id`; clearing cloud-init's +/// instance state makes it treat the clone as a new instance and re-run its +/// per-instance modules. +pub fn identity_reset_commands() -> Vec { + vec![ + "rm -f /etc/ssh/ssh_host_*".to_string(), + ": > /etc/machine-id".to_string(), + "rm -f /var/lib/dbus/machine-id".to_string(), + "rm -rf /var/lib/cloud/instances /var/lib/cloud/instance".to_string(), + "rm -f /root/.bash_history /home/*/.bash_history".to_string(), + "find /var/log -type f -exec truncate -s 0 {} + 2>/dev/null || true".to_string(), + ] +} + +/// Shell to stop the stack and drop every data volume that is not explicitly +/// preserved, so the buyer's box initializes them from scratch with the +/// buyer's own generated values. +/// +/// Volume names are matched by suffix because Compose prefixes them with the +/// project name (`project_stackpilot_pgdata` for a declared `stackpilot_pgdata`). +pub fn volume_reset_commands(project_dir: &str, keep: &[&str]) -> Vec { + let mut cmds = vec![format!( + "cd {project_dir} && docker compose down --remove-orphans" + )]; + + let filter = if keep.is_empty() { + "cat".to_string() + } else { + let pattern = keep.join("|"); + format!("grep -Ev '({pattern})'") + }; + + cmds.push(format!( + "docker volume ls -q | {filter} | xargs -r docker volume rm -f" + )); + cmds +} + +/// Blank the values of secret-bearing keys in a `.env` file while keeping the +/// file's shape: keys, comments, blank lines and non-secret values survive. +/// +/// The keys are kept (rather than the lines dropped) so the file still +/// documents what the stack expects; on the buyer's box the whole file is +/// replaced from `/etc/stacker/env` by the systemd unit's `ExecStartPre`, so +/// the blanked values are never read. +pub fn scrub_env_file(content: &str, protected: &BTreeSet) -> String { + let mut out = String::with_capacity(content.len()); + + for line in content.lines() { + let trimmed = line.trim_start(); + if trimmed.is_empty() || trimmed.starts_with('#') { + out.push_str(line); + out.push('\n'); + continue; + } + + match line.split_once('=') { + Some((key, _value)) if should_blank(key.trim(), protected) => { + out.push_str(key); + out.push_str("=\n"); + } + _ => { + out.push_str(line); + out.push('\n'); + } + } + } + + out +} + +/// A key is blanked when the author's contract declared it regenerable/buyer- +/// supplied, or when its name is secret-shaped by the same heuristic the CLI +/// already uses for `generate-secrets.sh`. +fn should_blank(key: &str, protected: &BTreeSet) -> bool { + protected.contains(key) || crate::console::commands::cli::init::is_secret_env_key(key) +} + +/// Everything the finalize step needs to reach and sanitize a build box. +#[derive(Debug, Clone)] +pub struct FinalizeContext { + pub host: String, + pub port: u16, + pub user: String, + pub private_key_pem: String, + /// Where the deploy put the compose file and its co-located `.env`. + pub project_dir: String, + /// Stack slug — selects the volume keep-list. + pub stack: String, + /// Contract fields with `mutability: generated`/`provided`. + pub protected_keys: BTreeSet, +} + +/// What the finalize step learned about the image it just sanitized. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FinalizeOutcome { + /// `${VAR}` names the sanitized compose references — pinned to the snapshot + /// so the clone path can fail closed on an environment that cannot satisfy + /// them. + pub required_env_keys: BTreeSet, +} + +/// Sanitize a build box in place, immediately before it is snapshotted. +/// +/// Order matters: the compose/env rewrite has to happen while the stack is +/// still described on disk, the volume reset tears the stack down, and the +/// identity reset goes last because it leaves the box unable to present a +/// stable SSH identity afterwards. The build box is throwaway, so none of this +/// needs to be reversible. +pub async fn finalize_build_box( + ctx: &FinalizeContext, +) -> Result { + use crate::helpers::bake::BakeError; + use crate::helpers::ssh_client::{disconnect_ssh, exec_remote, open_ssh}; + + let fail = |stage: &str, err: String| BakeError::Finalize(format!("{stage}: {err}")); + + let session = open_ssh( + &ctx.host, + ctx.port, + &ctx.user, + &ctx.private_key_pem, + std::time::Duration::from_secs(30), + ) + .await + .map_err(|e| fail("ssh connect", e.to_string()))?; + + let run = |cmd: String| { + let session = &session; + async move { + let (stdout, stderr, code) = exec_remote(session, &cmd, 300) + .await + .map_err(|e| e.to_string())?; + if code != 0 { + return Err(format!("`{cmd}` exited {code}: {stderr}")); + } + Ok::(stdout) + } + }; + + let compose_path = format!("{}/docker-compose.yml", ctx.project_dir); + let env_path = format!("{}/.env", ctx.project_dir); + + let result = async { + // 1. Read what the deploy left on the box. + let compose = run(format!("cat {compose_path}")) + .await + .map_err(|e| fail("read compose", e))?; + // A stack may legitimately have no .env; treat that as empty. + let env_raw = run(format!("cat {env_path} 2>/dev/null || true")) + .await + .map_err(|e| fail("read .env", e))?; + let env_values = parse_env_pairs(&env_raw); + + // 2. Replace secrets embedded inside larger values (DSNs) with ${KEY} + // references. Whole-value keys were already parameterized at deploy + // time by `parameterize_compose_env_vars`. + let sanitized = crate::cli::generator::compose::parameterize_embedded_secret_values( + &compose, + &env_values, + &ctx.protected_keys, + ) + .map_err(|conflict| BakeError::Finalize(conflict.to_string()))?; + + if sanitized != compose { + write_remote_file(&run, &compose_path, &sanitized) + .await + .map_err(|e| fail("write compose", e))?; + } + + // 3. Record what the image now needs from the buyer's env file. + let required_env_keys = + crate::cli::generator::compose::collect_env_var_references(&sanitized); + + // 4. Blank the author's secrets in the co-located .env. The buyer's box + // overwrites this file wholesale from /etc/stacker/env at boot, so + // the blanked values are never read. + if !env_raw.trim().is_empty() { + let scrubbed = scrub_env_file(&env_raw, &ctx.protected_keys); + write_remote_file(&run, &env_path, &scrubbed) + .await + .map_err(|e| fail("write .env", e))?; + } + + // 5. Drop initialized data volumes so the buyer's box re-initializes + // them with the buyer's own values. A secret the app persisted on + // first run is not governed by env vars any more. + for cmd in volume_reset_commands(&ctx.project_dir, volumes_to_keep(&ctx.stack)) { + run(cmd).await.map_err(|e| fail("volume reset", e))?; + } + + // 6. Strip machine identity last. + for cmd in identity_reset_commands() { + run(cmd).await.map_err(|e| fail("identity reset", e))?; + } + + Ok(FinalizeOutcome { required_env_keys }) + } + .await; + + disconnect_ssh(session).await; + result +} + +/// Write `content` to `path` on the remote box without any quoting hazards: +/// the payload travels base64-encoded and is decoded on the far side. +async fn write_remote_file(run: &F, path: &str, content: &str) -> Result<(), String> +where + F: Fn(String) -> Fut, + Fut: std::future::Future>, +{ + use base64::{engine::general_purpose::STANDARD, Engine as _}; + let encoded = STANDARD.encode(content.as_bytes()); + run(format!("printf %s {encoded} | base64 -d > {path}")) + .await + .map(|_| ()) +} + +/// Parse `KEY=value` lines into a map, skipping comments and blanks. +pub fn parse_env_pairs(content: &str) -> std::collections::BTreeMap { + let mut pairs = std::collections::BTreeMap::new(); + + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + if let Some((key, value)) = trimmed.split_once('=') { + let value = value.trim(); + if !value.is_empty() { + pairs.insert(key.trim().to_string(), value.to_string()); + } + } + } + + pairs +} + +/// The contract fields a buyer's box regenerates or supplies — i.e. the ones a +/// `${KEY}` reference can safely point at. +/// +/// Mirrors the selection `compose_env_keys` makes at deploy time +/// (`src/console/commands/cli/deploy.rs`), reading the contract as raw JSON so +/// an unparseable or partially-shaped contract degrades to "nothing protected" +/// rather than failing the bake outright. +pub fn protected_keys_from_contract(contract: &serde_json::Value) -> BTreeSet { + let mut keys = BTreeSet::new(); + + let Some(services) = contract.get("services").and_then(|v| v.as_object()) else { + return keys; + }; + + for service in services.values() { + let Some(fields) = service.get("fields").and_then(|v| v.as_object()) else { + continue; + }; + for (name, policy) in fields { + let protected = matches!( + policy.get("mutability").and_then(|v| v.as_str()), + Some("generated") | Some("provided") + ); + if protected { + keys.insert(name.clone()); + } + } + } + + keys +} + +#[cfg(test)] +mod tests { + use super::*; + + fn protected(keys: &[&str]) -> BTreeSet { + keys.iter().map(|k| k.to_string()).collect() + } + + #[test] + fn scrub_blanks_contract_declared_keys() { + let env = "SECRET_KEY=b838f1f2\nOLLAMA_MODEL=llama3.1\n"; + let out = scrub_env_file(env, &protected(["SECRET_KEY"].as_slice())); + assert!(out.contains("SECRET_KEY=\n"), "blanked:\n{out}"); + assert!( + out.contains("OLLAMA_MODEL=llama3.1"), + "non-secret kept:\n{out}" + ); + } + + #[test] + fn scrub_blanks_secret_shaped_keys_without_a_contract() { + // No contract at all — the name heuristic still has to catch these. + let env = "DB_PASSWORD=2213a996\nADMIN_USER=admin\n"; + let out = scrub_env_file(env, &BTreeSet::new()); + assert!(out.contains("DB_PASSWORD=\n"), "blanked:\n{out}"); + assert!(out.contains("ADMIN_USER=admin"), "non-secret kept:\n{out}"); + } + + #[test] + fn scrub_preserves_comments_and_blank_lines() { + let env = "# Secrets\n\nSECRET_KEY=abc\n"; + let out = scrub_env_file(env, &BTreeSet::new()); + assert_eq!(out, "# Secrets\n\nSECRET_KEY=\n"); + } + + #[test] + fn scrub_keeps_values_containing_equals_signs() { + // A blanked key must not be confused by '=' inside a kept value. + let env = "OLLAMA_MODEL=llama3.1\nJWT_SECRET=a=b=c\n"; + let out = scrub_env_file(env, &BTreeSet::new()); + assert!(out.contains("OLLAMA_MODEL=llama3.1"), "kept:\n{out}"); + assert!(out.contains("JWT_SECRET=\n"), "blanked:\n{out}"); + } + + #[test] + fn identity_reset_covers_host_keys_machine_id_and_cloud_init() { + let cmds = identity_reset_commands().join(" ; "); + assert!(cmds.contains("/etc/ssh/ssh_host_*"), "host keys: {cmds}"); + assert!(cmds.contains("/etc/machine-id"), "machine-id: {cmds}"); + assert!( + cmds.contains("/var/lib/cloud/instance"), + "cloud-init: {cmds}" + ); + } + + #[test] + fn volume_reset_preserves_the_kept_volumes() { + let cmds = volume_reset_commands("/home/trydirect/project", &["ollama"]).join(" ; "); + assert!( + cmds.contains("docker compose down"), + "stack stopped: {cmds}" + ); + assert!( + cmds.contains("grep -Ev '(ollama)'"), + "kept volume excluded from removal: {cmds}" + ); + } + + #[test] + fn volume_reset_without_a_keep_list_removes_everything() { + let cmds = volume_reset_commands("/home/trydirect/project", &[]).join(" ; "); + assert!(cmds.contains("| cat |"), "no filter applied: {cmds}"); + } + + #[test] + fn parse_env_pairs_skips_comments_blanks_and_empty_values() { + let env = "# header\n\nA=1\nEMPTY=\nB=two\n"; + let pairs = parse_env_pairs(env); + assert_eq!(pairs.get("A").map(String::as_str), Some("1")); + assert_eq!(pairs.get("B").map(String::as_str), Some("two")); + assert!(!pairs.contains_key("EMPTY"), "empty value is not a secret"); + } + + #[test] + fn protected_keys_cover_generated_and_provided_only() { + let contract = serde_json::json!({ + "services": { + "db": { "fields": { + "POSTGRES_PASSWORD": { "mutability": "generated" }, + "POSTGRES_USER": { "mutability": "fixed" } + }}, + "app": { "fields": { + "LICENSE_KEY": { "mutability": "provided" }, + "LOG_LEVEL": { "mutability": "editable" } + }} + } + }); + let keys = protected_keys_from_contract(&contract); + assert!(keys.contains("POSTGRES_PASSWORD")); + assert!(keys.contains("LICENSE_KEY")); + assert!(!keys.contains("POSTGRES_USER"), "fixed is not regenerated"); + assert!(!keys.contains("LOG_LEVEL"), "editable is not regenerated"); + } + + #[test] + fn protected_keys_from_a_shapeless_contract_is_empty() { + assert!(protected_keys_from_contract(&serde_json::Value::Null).is_empty()); + } + + #[test] + fn stackpilot_keeps_only_its_model_volume() { + assert_eq!(volumes_to_keep("stackpilot"), &["ollama"]); + // An unknown stack defaults to resetting everything — leaking is worse + // than rebuilding. + assert!(volumes_to_keep("something-else").is_empty()); + } +} diff --git a/src/helpers/mod.rs b/src/helpers/mod.rs index d741d19f..f071e56e 100644 --- a/src/helpers/mod.rs +++ b/src/helpers/mod.rs @@ -34,6 +34,7 @@ pub use dockerhub::*; pub use cloud::*; pub mod audit_cache; pub mod bake; +pub mod bake_finalize; pub mod bake_registry; pub mod cloud_init; pub mod compose_yaml; diff --git a/src/models/baked_snapshot.rs b/src/models/baked_snapshot.rs index 188b1e75..18f2a56a 100644 --- a/src/models/baked_snapshot.rs +++ b/src/models/baked_snapshot.rs @@ -19,6 +19,15 @@ pub struct BakedSnapshot { /// single value baked into the snapshot. `None` for snapshots baked before /// this column existed. pub config_contract: Option, + /// The `${VAR}` names the baked compose file references, captured at bake + /// time. The clone path checks the buyer's env against this before creating + /// a server: an unsatisfied reference resolves to an empty string at boot + /// with nothing but a Compose warning to show for it. + /// + /// `None` for snapshots baked before this column existed — those skip the + /// check rather than becoming undeployable. + #[sqlx(default)] + pub required_env_keys: Option, pub created_at: DateTime, } diff --git a/src/routes/oneclick_deploy/clone.rs b/src/routes/oneclick_deploy/clone.rs index 7f567f74..33b76a45 100644 --- a/src/routes/oneclick_deploy/clone.rs +++ b/src/routes/oneclick_deploy/clone.rs @@ -204,6 +204,44 @@ pub async fn clone_server( None => (Vec::new(), Vec::new()), }; + // ── fail closed on an env the image cannot boot with ───────────────── + // The bake pinned the `${VAR}` references its compose file actually needs. + // Compose resolves an unsatisfied reference to an *empty string* and only + // warns, and the unit's ExecStartPre ends in `|| true`, so a missing key + // produces a running-but-broken stack rather than a visible failure. Check + // before a server exists, so the buyer gets a refusal instead of a bill. + // + // Snapshots baked before `required_env_keys` carry NULL and skip this. + let provided: std::collections::BTreeSet = form + .env + .iter() + .filter(|(_, value)| !value.trim().is_empty()) + .map(|(key, _)| key.clone()) + .chain(regen.iter().map(|(key, _)| key.clone())) + .chain(regen_jwt.iter().map(|spec| spec.target_key.clone())) + .collect(); + + let missing = missing_required_env_keys(&snapshot.required_env_keys, &provided); + if !missing.is_empty() { + tracing::error!( + stack = %form.stack, + version = %snapshot.version, + missing = ?missing, + "refusing clone: baked compose references env keys the deploy would not supply" + ); + return HttpResponse::UnprocessableEntity().json(json!({ + "error": "Incomplete environment for this snapshot", + "details": format!( + "the baked image for '{}' v{} references {} environment variable(s) that this \ + deploy would not set ({}). They would resolve to empty strings at boot.", + form.stack, + snapshot.version, + missing.len(), + missing.join(", ") + ), + })); + } + // Render cloud-init with per-user env + domain. Secrets are pre-resolved (by // the user service) into `form.env`; `regen` mints fresh values for // `mutability: generated` fields on the box at first boot, reusing the same @@ -333,12 +371,11 @@ pub async fn clone_server( // template fallback stores a stack_definition blob that is not a // ProjectForm — parsing fails and the panel stays empty (the user can // add apps manually). - if let Ok(form) = serde_json::from_value::( - project.request_json.clone(), - ) { + if let Ok(form) = + serde_json::from_value::(project.request_json.clone()) + { if let Err(err) = - crate::project_app::sync_project_level_apps_from_form(&pg_pool, project.id, &form) - .await + crate::project_app::sync_project_level_apps_from_form(&pg_pool, project.id, &form).await { tracing::warn!( error = %err, @@ -652,6 +689,26 @@ pub async fn clone_server( }) } +/// The pinned `${VAR}` references the buyer's environment would leave unset. +/// +/// `required` is `baked_snapshots.required_env_keys` — a JSON array recorded at +/// bake time. `None` (a snapshot baked before that column) yields no misses, so +/// existing images keep deploying unchanged rather than becoming undeployable. +fn missing_required_env_keys( + required: &Option, + provided: &std::collections::BTreeSet, +) -> Vec { + let Some(serde_json::Value::Array(keys)) = required else { + return Vec::new(); + }; + + keys.iter() + .filter_map(serde_json::Value::as_str) + .filter(|key| !provided.contains(*key)) + .map(str::to_string) + .collect() +} + /// The generated fields whose fresh value must be minted on the cloned box, /// paired with the canonical shell generator for each. Reuses the *single* /// source of truth for the type→generator mapping @@ -744,13 +801,41 @@ fn derived_jwt_commands( #[cfg(test)] mod regen_tests { - use super::{derived_jwt_commands, regen_commands}; + use super::{derived_jwt_commands, missing_required_env_keys, regen_commands}; use serde_json::json; use std::collections::BTreeMap; /// A `generated` field with no installer-supplied value gets a regen command /// reusing the canonical shell generator; a `fixed` field and an /// already-supplied value are left alone. + #[test] + fn missing_required_env_keys_reports_only_unsatisfied_ones() { + let required = Some(json!(["ALPHA", "BETA", "GAMMA"])); + let provided: std::collections::BTreeSet = + ["ALPHA".to_string(), "GAMMA".to_string()] + .into_iter() + .collect(); + + assert_eq!( + missing_required_env_keys(&required, &provided), + vec!["BETA".to_string()] + ); + } + + /// Snapshots baked before the column must stay deployable. + #[test] + fn missing_required_env_keys_is_empty_for_legacy_snapshots() { + assert!(missing_required_env_keys(&None, &std::collections::BTreeSet::new()).is_empty()); + } + + #[test] + fn missing_required_env_keys_is_empty_when_all_supplied() { + let required = Some(json!(["POSTGRES_PASSWORD"])); + let provided: std::collections::BTreeSet = + ["POSTGRES_PASSWORD".to_string()].into_iter().collect(); + assert!(missing_required_env_keys(&required, &provided).is_empty()); + } + #[test] fn regen_commands_only_for_unset_generated_fields() { let contract: crate::cli::config_parser::ConfigContract = serde_json::from_value(json!({