From 10697ae899e5b3b26142581787250449448cb6ae Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Tue, 29 Sep 2026 21:41:39 -0400 Subject: [PATCH 01/25] fix: recover missing native provider HOME aliases --- docs/guides/native-candidate.md | 15 + .../runtime-core/src/provider/lifecycle.rs | 25 +- .../src/provider/lifecycle/short_home.rs | 70 +++++ .../provider/lifecycle/short_home/tests.rs | 265 ++++++++++++++++++ packages/runtime-core/src/provider/state.rs | 88 +++++- 5 files changed, 455 insertions(+), 8 deletions(-) create mode 100644 packages/runtime-core/src/provider/lifecycle/short_home.rs create mode 100644 packages/runtime-core/src/provider/lifecycle/short_home/tests.rs diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 7a31e89b0..2dbe382d3 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -118,6 +118,21 @@ complete bundle to change the candidate. binary refusing newer state is an unsupported downgrade, not a successful rollback. Preserve that state; do not rewrite receipts or adopt another home's data. +Host cleanup or a reboot can remove the temporary `/private/tmp/hkl-` HOME +alias while the private candidate home and disks remain intact. Normal commands +report `provider_home_missing`; they do not initialize another pool or recreate the +alias during observation. Explicit `runtime recover --json` can restore only that +absent exact alias, after verifying the receipt-bound dead provider, both recorded +disks, free VM lock, closed disk handles, and no active provider command. Existing +files, directories, foreign links, a live or reused PID, and uncertain ownership +refuse without replacement. Recovery rechecks the receipt before exclusive creation, +then validates socket absence and flushes the disks before recording +`recovered-unclean`. If those final checks fail, it reports +`provider_home_restored_recovery_incomplete`, keeps the exact owned alias and retained +data, and requires inspection before retry. No guest is started by alias recovery. +Interrupted starts without a recorded process or both identified disks remain +outside this repair path. + Compatibility is qualified for specific bundle hashes and state formats. The displayed version alone does not establish frontend/executor or downgrade compatibility. V4 and candidate homes remain separate; this procedure does not diff --git a/packages/runtime-core/src/provider/lifecycle.rs b/packages/runtime-core/src/provider/lifecycle.rs index 60e7109d3..8b47b97b0 100644 --- a/packages/runtime-core/src/provider/lifecycle.rs +++ b/packages/runtime-core/src/provider/lifecycle.rs @@ -3,6 +3,7 @@ mod prepared_boot; #[cfg(any(target_os = "macos", test))] mod private_child; mod relay_process; +mod short_home; use super::{ admission, agent, artifact, identity, process, state::{self, Owner, io}, @@ -1812,6 +1813,12 @@ fn finish_absent( value: &str, record_disks: bool, ) -> Result<(), CandidateError> { + let vm_lock = lock_absent_disks(candidate, owner)?; + finish_absent_locked(candidate, owner, value, record_disks, &vm_lock) +} + +/// Keep this descriptor through alias restoration and the stopped receipt. +fn lock_absent_disks(candidate: &Candidate, owner: &Owner) -> Result { let directory = owner.real_data_dir(candidate)?; let vm_lock = OpenOptions::new() .read(true) @@ -1847,6 +1854,16 @@ fn finish_absent( )); } verify_disks(candidate, owner)?; + Ok(vm_lock) +} + +fn finish_absent_locked( + candidate: &Candidate, + owner: &mut Owner, + value: &str, + record_disks: bool, + _vm_lock: &File, +) -> Result<(), CandidateError> { if record_disks && owner.storage.is_none() { owner.storage = Some(identity::disk( &owner.real_data_dir(candidate)?.join("storage.raw"), @@ -1896,7 +1913,13 @@ fn finish_absent( } pub fn recover(candidate: &Candidate) -> Result { - let initial = status(candidate)?; + let initial = match status(candidate) { + Ok(initial) => initial, + Err(error) if error.code == "provider_home_missing" => { + return short_home::recover(candidate); + } + Err(error) => return Err(error), + }; if initial.phase == "uninitialized" { return Ok(initial); } diff --git a/packages/runtime-core/src/provider/lifecycle/short_home.rs b/packages/runtime-core/src/provider/lifecycle/short_home.rs new file mode 100644 index 000000000..6678ab602 --- /dev/null +++ b/packages/runtime-core/src/provider/lifecycle/short_home.rs @@ -0,0 +1,70 @@ +//! Explicit two-phase recovery of the receipt-bound temporary HOME after host cleanup. +//! +//! No provider is launched and no existing alias is replaced. Durable disks and VM absence +//! are audited before restoring the short socket path; socket absence is then checked before +//! committing the recovered phase. A later refusal retains the exact alias and unchanged data. +use super::{ + Owner, RuntimeStatus, binary, finish_absent_locked, identity, lock_absent_disks, root, state, + status, verify_disks, +}; +use crate::{Candidate, CandidateError}; + +fn dead_provider(candidate: &Candidate, owner: &Owner) -> Result<(), CandidateError> { + let process = owner.process.as_ref().ok_or_else(|| { + CandidateError::new( + "recovery_required", + "Missing HOME recovery requires a recorded provider identity; nothing was changed.", + ) + })?; + // SAFETY: geteuid has no preconditions. + identity::verify(process, process, &binary(candidate), unsafe { + libc::geteuid() + })?; + if identity::alive(process.pid)? || identity::executable_running(&binary(candidate))? { + return Err(CandidateError::new( + "recovery_required", + "Missing HOME recovery requires a confirmed dead provider and no active provider command; nothing was changed.", + )); + } + if !owner.created || owner.storage.is_none() || owner.overlay.is_none() { + return Err(CandidateError::new( + "recovery_required", + "Missing HOME recovery requires both previously identified disks; nothing was adopted.", + )); + } + Ok(()) +} + +pub(super) fn recover(candidate: &Candidate) -> Result { + let _operation = state::Lock::acquire_existing(&root(candidate))?; + let mut owner = Owner::load_for_short_home_recovery(candidate)?; + dead_provider(candidate, &owner)?; + let vm_lock = lock_absent_disks(candidate, &owner)?; + // Recheck immediately before the only new external effect. The locks fence cooperative + // writers; no signal, disk adoption, overwrite or receipt update happens in this phase. + dead_provider(candidate, &owner)?; + verify_disks(candidate, &owner)?; + owner.restore_missing_short_home(candidate)?; + let mut finish = || -> Result<(), CandidateError> { + let observed = Owner::load(candidate)?; + if observed != owner { + return Err(CandidateError::new( + "foreign_state", + "Provider receipt changed after HOME restoration.", + )); + } + dead_provider(candidate, &owner)?; + verify_disks(candidate, &owner)?; + finish_absent_locked(candidate, &mut owner, "recovered-unclean", false, &vm_lock) + }; + finish().map_err(|error| { + CandidateError::new( + "provider_home_restored_recovery_incomplete", + format!("Owned HOME alias restored; runtime recovery is incomplete ({}). Data retained; inspect and retry runtime recover.", error.code), + ) + })?; + status(candidate) +} + +#[cfg(all(test, target_os = "macos"))] +mod tests; diff --git a/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs b/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs new file mode 100644 index 000000000..f22b62192 --- /dev/null +++ b/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs @@ -0,0 +1,265 @@ +use super::*; +use crate::provider::{NetworkIntent, Profile}; +use std::{ + fs, + os::{ + fd::AsRawFd, + unix::{fs::MetadataExt, net::UnixListener}, + }, + path::PathBuf, + process::Command, +}; + +struct Pool { + candidate: Candidate, + owner: Owner, + directory: PathBuf, +} +impl Pool { + fn new() -> Self { + let mut child = Command::new("/bin/sleep").arg("30").spawn().unwrap(); + let mut process = identity::observe(child.id() as i32).unwrap(); + child.kill().unwrap(); + child.wait().unwrap(); + let directory = fs::canonicalize(std::env::temp_dir()) + .unwrap() + .join(format!( + "hack-home-recovery-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + state::private_directory(&directory).unwrap(); + let candidate = Candidate::discover(&directory).unwrap(); + let operation = state::Lock::acquire(&root(&candidate)).unwrap(); + let mut owner = Owner::create( + &candidate, + Profile::Development, + None, + NetworkIntent::Isolated, + ) + .unwrap(); + let data = owner.real_data_dir(&candidate).unwrap(); + state::private_directory(&data).unwrap(); + for (name, tag) in [("storage.raw", 1u8), ("overlay.raw", 2)] { + let mut bytes = vec![0u8; 4096]; + bytes[1080..1082].copy_from_slice(&[0x53, 0xef]); + bytes[1128..1144].fill(tag); + bytes[2048..2059].copy_from_slice(b"data-marker"); + fs::write(data.join(name), bytes).unwrap(); + } + fs::write(data.join("vm.lock"), b"").unwrap(); + fs::write(data.join("name"), &owner.machine).unwrap(); + process.executable = binary(&candidate); + owner.process = Some(process); + owner.created = true; + owner.phase = "running".into(); + owner.storage = Some(identity::disk(&data.join("storage.raw")).unwrap()); + owner.overlay = Some(identity::disk(&data.join("overlay.raw")).unwrap()); + owner.save(&candidate).unwrap(); + drop(operation); + Self { + candidate, + owner, + directory, + } + } + fn remove_alias(&self) { + fs::remove_file(&self.owner.short_home).unwrap(); + } + fn receipt(&self) -> Vec { + fs::read(root(&self.candidate).join("owner.json")).unwrap() + } + fn data(&self) -> PathBuf { + self.owner.real_data_dir(&self.candidate).unwrap() + } + fn unchanged(&self, before: &[u8]) { + assert_eq!(self.receipt(), before); + assert!(self.owner.short_home.symlink_metadata().is_err()); + } +} +impl Drop for Pool { + fn drop(&mut self) { + let _ = fs::remove_file(&self.owner.short_home); + fs::remove_dir_all(&self.directory).unwrap(); + } +} + +#[test] +fn explicit_recovery_restores_only_missing_alias_and_preserves_disk_bytes() { + let pool = Pool::new(); + let storage = fs::read(pool.data().join("storage.raw")).unwrap(); + let overlay = fs::read(pool.data().join("overlay.raw")).unwrap(); + pool.remove_alias(); + assert_eq!( + status(&pool.candidate).unwrap_err().code, + "provider_home_missing" + ); + let recovered = crate::provider::recover(&pool.candidate).unwrap(); + assert_eq!(recovered.phase, "recovered-unclean"); + assert_eq!(recovered.process_alive, Some(false)); + assert_eq!( + fs::read_link(&pool.owner.short_home).unwrap(), + root(&pool.candidate).join("home") + ); + assert_eq!(fs::read(pool.data().join("storage.raw")).unwrap(), storage); + assert_eq!(fs::read(pool.data().join("overlay.raw")).unwrap(), overlay); + assert_eq!( + Owner::load(&pool.candidate).unwrap().storage, + pool.owner.storage + ); +} + +#[test] +fn live_or_reused_pid_and_missing_disk_identity_refuse_without_alias_effect() { + for case in 0..3 { + let mut pool = Pool::new(); + if case < 2 { + let p = pool.owner.process.as_mut().unwrap(); + p.pid = std::process::id() as i32; + p.start_micros = if case == 0 { + identity::observe(p.pid).unwrap().start_micros + } else { + 1 + }; + } else { + pool.owner.overlay = None; + } + pool.owner.save(&pool.candidate).unwrap(); + pool.remove_alias(); + let before = pool.receipt(); + assert_eq!( + crate::provider::recover(&pool.candidate).unwrap_err().code, + "recovery_required" + ); + pool.unchanged(&before); + } +} + +#[test] +fn changed_disk_held_vm_lock_and_open_disk_refuse_before_alias_creation() { + for case in 0..3 { + let pool = Pool::new(); + pool.remove_alias(); + let before = pool.receipt(); + let mut held = None; + if case == 0 { + fs::write(pool.data().join("overlay.raw"), b"changed disk").unwrap(); + } + if case == 1 { + let f = fs::File::open(pool.data().join("vm.lock")).unwrap(); + assert_eq!( + unsafe { libc::flock(f.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }, + 0 + ); + held = Some(f); + } + if case == 2 { + held = Some(fs::File::open(pool.data().join("storage.raw")).unwrap()); + } + assert!(crate::provider::recover(&pool.candidate).is_err()); + pool.unchanged(&before); + drop(held); + } +} + +#[test] +fn existing_file_directory_and_foreign_symlink_are_never_replaced() { + for case in 0..3 { + let pool = Pool::new(); + pool.remove_alias(); + let before = pool.receipt(); + if case == 0 { + fs::write(&pool.owner.short_home, b"foreign marker").unwrap(); + } + if case == 1 { + fs::create_dir(&pool.owner.short_home).unwrap(); + } + if case == 2 { + std::os::unix::fs::symlink(&pool.directory, &pool.owner.short_home).unwrap(); + } + let inode = fs::symlink_metadata(&pool.owner.short_home).unwrap().ino(); + assert_eq!( + crate::provider::recover(&pool.candidate).unwrap_err().code, + "foreign_state" + ); + assert_eq!(pool.receipt(), before); + assert_eq!( + fs::symlink_metadata(&pool.owner.short_home).unwrap().ino(), + inode + ); + if case == 1 { + fs::remove_dir(&pool.owner.short_home).unwrap(); + } + } +} + +#[test] +fn active_socket_leaves_explicit_incomplete_recovery_then_retry_finishes() { + let pool = Pool::new(); + let listener = UnixListener::bind(pool.owner.data_dir().join("agent.sock")).unwrap(); + pool.remove_alias(); + let before = pool.receipt(); + let error = crate::provider::recover(&pool.candidate).unwrap_err(); + assert_eq!(error.code, "provider_home_restored_recovery_incomplete"); + assert!(error.message.contains("stop_uncertain")); + assert_eq!(pool.receipt(), before); + assert_eq!( + fs::read_link(&pool.owner.short_home).unwrap(), + root(&pool.candidate).join("home") + ); + drop(listener); + assert_eq!( + crate::provider::recover(&pool.candidate).unwrap().phase, + "recovered-unclean" + ); +} + +#[test] +fn receipt_change_and_concurrent_alias_creation_refuse_exclusive_repair() { + let mut pool = Pool::new(); + pool.remove_alias(); + let observed = Owner::load_for_short_home_recovery(&pool.candidate).unwrap(); + pool.owner.phase = "stopped".into(); + pool.owner.save(&pool.candidate).unwrap(); + let before = pool.receipt(); + assert_eq!( + observed + .restore_missing_short_home(&pool.candidate) + .unwrap_err() + .code, + "foreign_state" + ); + pool.unchanged(&before); + std::os::unix::fs::symlink(root(&pool.candidate).join("home"), &pool.owner.short_home).unwrap(); + let inode = fs::symlink_metadata(&pool.owner.short_home).unwrap().ino(); + assert_eq!( + pool.owner + .restore_missing_short_home(&pool.candidate) + .unwrap_err() + .code, + "socket_alias_collision" + ); + assert_eq!( + fs::symlink_metadata(&pool.owner.short_home).unwrap().ino(), + inode + ); + assert_eq!(pool.receipt(), before); +} + +#[test] +fn pending_owner_update_is_preserved_and_refuses_alias_repair() { + let pool = Pool::new(); + pool.remove_alias(); + let before = pool.receipt(); + let pending = root(&pool.candidate).join("owner.pending"); + fs::write(&pending, b"interrupted update").unwrap(); + assert_eq!( + crate::provider::recover(&pool.candidate).unwrap_err().code, + "recovery_required" + ); + pool.unchanged(&before); + assert_eq!(fs::read(&pending).unwrap(), b"interrupted update"); +} diff --git a/packages/runtime-core/src/provider/state.rs b/packages/runtime-core/src/provider/state.rs index 7f3b1a81a..e8810dbe1 100644 --- a/packages/runtime-core/src/provider/state.rs +++ b/packages/runtime-core/src/provider/state.rs @@ -57,7 +57,6 @@ impl Lock { } /// Observe existing state without initializing a directory or lock file. - #[cfg(target_os = "macos")] pub fn acquire_existing(root: &Path) -> Result { check_private_directory(root)?; let file = OpenOptions::new() @@ -177,7 +176,7 @@ impl Default for ReclamationPolicy { } } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Owner { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -211,8 +210,34 @@ pub struct Owner { } impl Owner { pub fn load(candidate: &Candidate) -> Result { + Self::load_with_short_home(candidate, false) + } + + /// Explicit recovery may inspect a missing temporary alias, never a replaced one. + /// Loading here is read-only; the lifecycle boundary still proves provider absence. + pub(super) fn load_for_short_home_recovery( + candidate: &Candidate, + ) -> Result { + Self::load_with_short_home(candidate, true) + } + + fn load_with_short_home( + candidate: &Candidate, + allow_missing: bool, + ) -> Result { let root = candidate.state_root.join("run/smolvm"); reject_aliased_state(&root)?; + if allow_missing { + match fs::symlink_metadata(root.join("owner.pending")) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + _ => { + return Err(CandidateError::new( + "recovery_required", + "Provider owner update is pending or unobservable; HOME alias was not restored.", + )); + } + } + } let owner: Self = read(&root.join("owner.json"))?; owner.network.validate()?; if let Some(share) = &owner.project_share { @@ -250,20 +275,69 @@ impl Owner { "Provider owner identity does not match this checkout.", )); } - // The only permitted alias is an exact, receipt-bound short HOME for Unix sockets. - let alias = fs::symlink_metadata(&owner.short_home).map_err(io)?; + owner.check_short_home(candidate, allow_missing)?; + check_private_directory(&root.join("home"))?; + Ok(owner) + } + /// The only permitted alias is the exact receipt-bound HOME for Unix sockets. + fn check_short_home( + &self, + candidate: &Candidate, + allow_missing: bool, + ) -> Result<(), CandidateError> { + let alias = match fs::symlink_metadata(&self.short_home) { + Ok(alias) => alias, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return if allow_missing { + Ok(()) + } else { + Err(CandidateError::new( + "provider_home_missing", + "Temporary provider HOME alias is absent. Use runtime recover to verify the stopped pool before restoring it.", + )) + }; + } + Err(error) => return Err(io(error)), + }; if !alias.file_type().is_symlink() || alias.uid() != unsafe { libc::geteuid() } - || fs::read_link(&owner.short_home).map_err(io)? != root.join("home") + || fs::read_link(&self.short_home).map_err(io)? + != candidate.state_root.join("run/smolvm/home") { return Err(CandidateError::new( "foreign_state", "Short provider HOME alias was replaced.", )); } - check_private_directory(&root.join("home"))?; - Ok(owner) + Ok(()) } + + /// Called only while the lifecycle owns the pool and VM locks and has audited absence. + /// Recheck the durable selection, then create exclusively; a concurrent alias is retained. + pub(super) fn restore_missing_short_home( + &self, + candidate: &Candidate, + ) -> Result<(), CandidateError> { + let observed = Self::load_for_short_home_recovery(candidate)?; + if observed != *self { + return Err(CandidateError::new( + "foreign_state", + "Provider owner changed during HOME recovery; no alias was restored.", + )); + } + std::os::unix::fs::symlink( + candidate.state_root.join("run/smolvm/home"), + &self.short_home, + ) + .map_err(|_| { + CandidateError::new( + "socket_alias_collision", + "Cannot exclusively restore provider HOME alias; no existing path was replaced.", + ) + })?; + self.check_short_home(candidate, false) + } + #[cfg(all(test, target_os = "macos"))] pub fn create( candidate: &Candidate, From 8f99174def8cd7cdd5e0d0187582e5570564f892 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Tue, 29 Sep 2026 22:12:49 -0400 Subject: [PATCH 02/25] fix: explicitly recover legacy host filesystem device identities --- docs/guides/native-candidate.md | 32 ++ packages/runtime-core/src/main.rs | 28 ++ .../runtime-core/src/provider/lifecycle.rs | 1 + .../src/provider/lifecycle/host_filesystem.rs | 267 ++++++++++++++++ .../lifecycle/host_filesystem/tests.rs | 294 ++++++++++++++++++ packages/runtime-core/src/provider/mod.rs | 1 + packages/runtime-core/src/provider/state.rs | 2 +- 7 files changed, 624 insertions(+), 1 deletion(-) create mode 100644 packages/runtime-core/src/provider/lifecycle/host_filesystem.rs create mode 100644 packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 2dbe382d3..752281131 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -133,6 +133,38 @@ data, and requires inspection before retry. No guest is started by alias recover Interrupted starts without a recorded process or both identified disks remain outside this repair path. +A physical macOS reboot may also renumber the mounted filesystem device. Strict +disk and source checks still refuse a changed device number; missing-HOME recovery +does not waive them. For an offline **stock pool**, inspect the separate migration: + +```sh +./hack-native --candidate-root /absolute/private/candidate-home runtime host-filesystem-recovery --json +./hack-native --candidate-root /absolute/private/candidate-home runtime recover-host-filesystem --expect-sha256 --accept-legacy-device-rebind --json +./hack-native --candidate-root /absolute/private/candidate-home runtime recover --json +``` + +Review the inspection before selecting its hash. This explicit legacy migration +requires an absent recorded provider whose start predates the current host boot, +no active provider commands, exclusive existing operation/VM locks and closed disk +handles. Both disk inodes, sizes and ext4 UUIDs, the exact source path/inode and its +ownership must be unchanged; only one common old-to-new device-number change is +allowed. The inspection is read-only. Publication atomically changes only the +owner's disk and source device numbers, retaining its phase and process record. +Normal commands keep their strict identity checks. + +Legacy receipts have no original host boot UUID or filesystem volume UUID. Calendar +timestamps corroborate a reboot, and matching retained file identities constrain +the migration, but neither proves original volume continuity. The opt-in explicitly +accepts that limitation; copied or relocated pools are outside this procedure. +Prepared-base pools and pending owner/network/activation updates require separate +recovery and are refused. A torn owner publication preserves `owner.pending` and +blocks another migration; do not delete or adopt that file manually. + +This does not start the VM, restore HOME, retire stale sockets or rewrite historical +graph receipts. Use ordinary recovery afterward. Historical shared-source graphs +retain the prior identity and may require verified cleanup plus a new generation; +successful metadata migration alone does not establish application recovery. + Compatibility is qualified for specific bundle hashes and state formats. The displayed version alone does not establish frontend/executor or downgrade compatibility. V4 and candidate homes remain separate; this procedure does not diff --git a/packages/runtime-core/src/main.rs b/packages/runtime-core/src/main.rs index 68bdc05ed..be6ee57a7 100644 --- a/packages/runtime-core/src/main.rs +++ b/packages/runtime-core/src/main.rs @@ -86,6 +86,8 @@ Usage: hack-local runtime up --profile development --internet --json hack-local runtime network extend --allow-host [--allow-host ] --json hack-local runtime up|status|down|recover [--json] + hack-local runtime host-filesystem-recovery [--json] + hack-local runtime recover-host-filesystem --expect-sha256 --accept-legacy-device-rebind [--json] hack-local node serve|status|inspect hack-local node request hack-local --version @@ -857,6 +859,32 @@ fn run() -> Result<(), CandidateError> { image, )?)?; } + ["runtime", "host-filesystem-recovery"] + | ["runtime", "host-filesystem-recovery", "--json"] => { + print_json(&hack_runtime_core::provider::host_filesystem::inspect( + &discover_candidate(&requested)?, + )?)?; + } + [ + "runtime", + "recover-host-filesystem", + "--expect-sha256", + hash, + "--accept-legacy-device-rebind", + ] + | [ + "runtime", + "recover-host-filesystem", + "--expect-sha256", + hash, + "--accept-legacy-device-rebind", + "--json", + ] => { + print_json(&hack_runtime_core::provider::host_filesystem::recover( + &discover_candidate(&requested)?, + hash, + )?)?; + } ["runtime", action] | ["runtime", action, "--json"] => { let candidate = discover_candidate(&requested)?; let result = match *action { diff --git a/packages/runtime-core/src/provider/lifecycle.rs b/packages/runtime-core/src/provider/lifecycle.rs index 8b47b97b0..36decce95 100644 --- a/packages/runtime-core/src/provider/lifecycle.rs +++ b/packages/runtime-core/src/provider/lifecycle.rs @@ -1,3 +1,4 @@ +pub mod host_filesystem; mod interrupted; mod prepared_boot; #[cfg(any(target_os = "macos", test))] diff --git a/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs b/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs new file mode 100644 index 000000000..08467b8f3 --- /dev/null +++ b/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs @@ -0,0 +1,267 @@ +//! Explicit legacy device-number migration for an offline stock pool. +//! +//! Old receipts have neither a host boot UUID nor a filesystem volume UUID. Calendar start +//! time, unchanged inode/size/ext4 UUID and exact paths constrain this migration, but cannot +//! establish original volume continuity. Only an explicitly accepted, hash-selected inspection +//! may update the owner. Normal runtime operations retain their strict identity comparisons. +use super::{Owner, binary, identity, lock_absent_disks, root, state}; +use crate::{Candidate, CandidateError}; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::{fs, os::unix::fs::MetadataExt}; + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Inspection { + pub schema: &'static str, + pub selection_sha256: String, + pub owner_sha256: String, + pub machine: String, + pub host_boot_micros: u64, + pub provider_start_micros: u64, + pub old_device: u64, + pub new_device: u64, + pub pool_inode: u64, + pub storage: identity::DiskIdentity, + pub overlay: identity::DiskIdentity, + pub project_share: Option, + pub qualification: &'static str, +} + +fn refused(detail: &str) -> CandidateError { + CandidateError::new( + "host_filesystem_recovery", + format!("{detail}; no identity was changed."), + ) +} + +#[cfg(target_os = "macos")] +fn host_boot_micros() -> Result { + let mut time = std::mem::MaybeUninit::::zeroed(); + let mut length = std::mem::size_of::(); + // SAFETY: read-only sysctl writes an exactly sized timeval; no input or retained pointers. + if unsafe { + libc::sysctlbyname( + c"kern.boottime".as_ptr(), + time.as_mut_ptr().cast(), + &mut length, + std::ptr::null_mut(), + 0, + ) + } != 0 + || length != std::mem::size_of::() + { + return Err(refused("Native host boot time is unavailable")); + } + // SAFETY: a complete timeval was returned above. + let time = unsafe { time.assume_init() }; + let seconds = u64::try_from(time.tv_sec).map_err(|_| refused("Invalid host boot time"))?; + let micros = u64::try_from(time.tv_usec).map_err(|_| refused("Invalid host boot time"))?; + if micros >= 1_000_000 { + return Err(refused("Invalid host boot time")); + } + seconds + .checked_mul(1_000_000) + .and_then(|v| v.checked_add(micros)) + .filter(|v| *v > 0) + .ok_or_else(|| refused("Invalid host boot time")) +} + +#[cfg(not(target_os = "macos"))] +fn host_boot_micros() -> Result { + Err(CandidateError::new( + "unsupported_host", + "Legacy filesystem recovery requires macOS.", + )) +} + +fn no_auxiliary_update(candidate: &Candidate) -> Result<(), CandidateError> { + crate::provider::network_update::require_complete(candidate)?; + for name in [ + "owner.pending", + "prepared-base.json", + "prepared-base.json.pending", + ] { + match fs::symlink_metadata(root(candidate).join(name)) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + _ => { + return Err(refused( + "Pending owner or prepared-base state requires separate recovery", + )); + } + } + } + Ok(()) +} + +fn dead_provider(candidate: &Candidate, owner: &Owner, boot: u64) -> Result<(), CandidateError> { + let process = owner + .process + .as_ref() + .ok_or_else(|| refused("No recorded provider identity"))?; + // SAFETY: geteuid has no preconditions. + identity::verify(process, process, &binary(candidate), unsafe { + libc::geteuid() + })?; + if !owner.created + || process.start_micros >= boot + || identity::alive(process.pid)? + || identity::executable_running(&binary(candidate))? + { + return Err(refused( + "Provider must be absent and its recorded start must predate this host boot", + )); + } + Ok(()) +} + +fn same_disk(before: &identity::DiskIdentity, after: &identity::DiskIdentity) -> bool { + before.device != after.device + && before.inode == after.inode + && before.bytes == after.bytes + && before.uuid == after.uuid +} + +/// Build a candidate owner in memory. This function neither adopts a new disk nor writes state. +fn selection( + candidate: &Candidate, + owner: &Owner, + boot: u64, +) -> Result<(Inspection, Owner), CandidateError> { + no_auxiliary_update(candidate)?; + dead_provider(candidate, owner, boot)?; + let data = owner.real_data_dir(candidate)?; + let storage = identity::disk(&data.join("storage.raw"))?; + let overlay = identity::disk(&data.join("overlay.raw"))?; + let before = owner + .storage + .as_ref() + .ok_or_else(|| refused("Storage identity is missing"))?; + let prior_overlay = owner + .overlay + .as_ref() + .ok_or_else(|| refused("Overlay identity is missing"))?; + let metadata = fs::symlink_metadata(root(candidate)).map_err(state::io)?; + if !same_disk(before, &storage) + || !same_disk(prior_overlay, &overlay) + || before.device != prior_overlay.device + || storage.device != overlay.device + || storage.device != metadata.dev() + { + return Err(refused( + "Only a common device-number change with unchanged disk inode, size and UUID is accepted", + )); + } + let mut next = owner.clone(); + if let Some(share) = &owner.project_share { + let observed = + crate::provider::ProjectShareIntent::approve(&share.project, share.unfiltered_source)?; + let mut expected = share.clone(); + expected.device = storage.device; + if share.device != before.device || observed != expected { + return Err(refused( + "Project share changed beyond the same filesystem device number", + )); + } + next.project_share = Some(observed); + } + next.storage = Some(storage.clone()); + next.overlay = Some(overlay.clone()); + // The canonical read is bounded and validated independently of raw-byte hashing. + let bytes = crate::provider::prepared_base::read_private( + &root(candidate).join("owner.json"), + 1024 * 1024, + )?; + if Owner::load_for_short_home_recovery(candidate)? != *owner { + return Err(refused("Owner selection changed")); + } + let mut inspection = Inspection { + schema: "hack.host-filesystem-recovery/v1", + selection_sha256: String::new(), + owner_sha256: format!("{:x}", Sha256::digest(bytes)), + machine: owner.machine.clone(), + host_boot_micros: boot, + provider_start_micros: owner + .process + .as_ref() + .ok_or_else(|| refused("No process"))? + .start_micros, + old_device: before.device, + new_device: storage.device, + pool_inode: metadata.ino(), + storage, + overlay, + project_share: next.project_share.clone(), + qualification: "explicit-legacy-migration-original-volume-continuity-unproven", + }; + inspection.selection_sha256 = format!( + "{:x}", + Sha256::digest( + serde_json::to_vec(&inspection).map_err(|_| refused("Cannot encode selection"))? + ) + ); + Ok((inspection, next)) +} + +/// Read-only inspection holds both existing locks and proves disks have no open handles. +pub fn inspect(candidate: &Candidate) -> Result { + let _operation = state::Lock::acquire_existing(&root(candidate))?; + let owner = Owner::load_for_short_home_recovery(candidate)?; + let (inspection, next) = selection(candidate, &owner, host_boot_micros()?)?; + let _vm = lock_absent_disks(candidate, &next)?; + if selection(candidate, &owner, host_boot_micros()?)?.0 != inspection { + return Err(refused("Inspection changed during absence verification")); + } + Ok(inspection) +} + +/// Publish disk and source device changes in ONE atomic owner replacement. A crash leaves +/// old or new committed metadata; any unfinished owner.pending is preserved and blocks retry. +/// This does not launch, restore the alias, retire historical sockets, or rewrite graph receipts. +pub fn recover(candidate: &Candidate, expected: &str) -> Result { + recover_with_boot(candidate, expected, host_boot_micros) +} + +fn recover_with_boot( + candidate: &Candidate, + expected: &str, + boot: impl Fn() -> Result, +) -> Result { + if expected.len() != 64 || !expected.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(refused("An exact inspection SHA-256 is required")); + } + let _operation = state::Lock::acquire_existing(&root(candidate))?; + let owner = Owner::load_for_short_home_recovery(candidate)?; + let (inspection, next) = selection(candidate, &owner, boot()?)?; + if inspection.selection_sha256 != expected { + return Err(refused("Inspection selection is stale")); + } + let _vm = lock_absent_disks(candidate, &next)?; + if selection(candidate, &owner, boot()?)?.0 != inspection { + return Err(refused("Selected identities changed before publication")); + } + next.save(candidate)?; + if Owner::load_for_short_home_recovery(candidate)? != next { + return Err(CandidateError::new( + "host_filesystem_recovery_incomplete", + "Owner was published but changed during confirmation; retained state requires inspection.", + )); + } + super::verify_disks(candidate, &next).map_err(|e| { + CandidateError::new( + "host_filesystem_recovery_incomplete", + format!( + "Owner was published but disk confirmation failed ({}); inspect retained state.", + e.code + ), + ) + })?; + if let Some(share) = &next.project_share { + share.validate().map_err(|e| CandidateError::new( + "host_filesystem_recovery_incomplete", format!("Owner was published but source confirmation failed ({}); inspect retained state.", e.code) + ))?; + } + Ok(inspection) +} + +#[cfg(all(test, target_os = "macos"))] +mod tests; diff --git a/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs b/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs new file mode 100644 index 000000000..3585833bf --- /dev/null +++ b/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs @@ -0,0 +1,294 @@ +use super::*; +use crate::provider::{NetworkIntent, Profile, ProjectShareIntent}; +use std::{fs::File, os::fd::AsRawFd, path::PathBuf, process::Command}; + +struct Pool { + candidate: Candidate, + owner: Owner, + directory: PathBuf, + boot: u64, +} +impl Pool { + fn new() -> Self { + let mut child = Command::new("/bin/sleep").arg("30").spawn().unwrap(); + let mut process = identity::observe(child.id() as i32).unwrap(); + child.kill().unwrap(); + child.wait().unwrap(); + let boot = process.start_micros + 1; + let directory = fs::canonicalize(std::env::temp_dir()) + .unwrap() + .join(format!( + "hack-device-recovery-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + state::private_directory(&directory).unwrap(); + let candidate = Candidate::discover(&directory).unwrap(); + let operation = state::Lock::acquire(&root(&candidate)).unwrap(); + let project = directory.join("app"); + state::private_directory(&project).unwrap(); + fs::write(project.join("package.json"), b"{}").unwrap(); + let share = ProjectShareIntent::approve(&project, true).unwrap(); + let mut owner = Owner::create_with_project_share( + &candidate, + Profile::Development, + None, + NetworkIntent::Isolated, + None, + Some(share), + ) + .unwrap(); + let data = owner.real_data_dir(&candidate).unwrap(); + state::private_directory(&data).unwrap(); + for (name, tag) in [("storage.raw", 1u8), ("overlay.raw", 2)] { + let mut bytes = vec![0; 4096]; + bytes[1080..1082].copy_from_slice(&[0x53, 0xef]); + bytes[1128..1144].fill(tag); + bytes[2048..2059].copy_from_slice(b"data-marker"); + fs::write(data.join(name), bytes).unwrap(); + } + fs::write(data.join("name"), &owner.machine).unwrap(); + fs::write(data.join("vm.lock"), b"").unwrap(); + process.executable = binary(&candidate); + owner.process = Some(process); + owner.created = true; + owner.phase = "running".into(); + let mut storage = identity::disk(&data.join("storage.raw")).unwrap(); + let mut overlay = identity::disk(&data.join("overlay.raw")).unwrap(); + let old = storage.device + 1; + storage.device = old; + overlay.device = old; + owner.storage = Some(storage); + owner.overlay = Some(overlay); + owner.project_share.as_mut().unwrap().device = old; + owner.save(&candidate).unwrap(); + fs::remove_file(&owner.short_home).unwrap(); + drop(operation); + Self { + candidate, + owner, + directory, + boot, + } + } + fn data(&self) -> PathBuf { + self.owner.real_data_dir(&self.candidate).unwrap() + } + fn selected(&self) -> Inspection { + selection(&self.candidate, &self.owner, self.boot) + .unwrap() + .0 + } + fn receipt(&self) -> Vec { + fs::read(root(&self.candidate).join("owner.json")).unwrap() + } + fn apply(&self, hash: &str) -> Result { + recover_with_boot(&self.candidate, hash, || Ok(self.boot)) + } + fn unchanged(&self, before: &[u8]) { + assert_eq!(self.receipt(), before); + assert!(self.owner.short_home.symlink_metadata().is_err()); + } +} +impl Drop for Pool { + fn drop(&mut self) { + let _ = fs::remove_file(&self.owner.short_home); + fs::remove_dir_all(&self.directory).unwrap(); + } +} + +#[test] +fn migration_changes_only_devices_and_keeps_alias_graph_history_and_data() { + let pool = Pool::new(); + let old = pool.receipt(); + let storage = fs::read(pool.data().join("storage.raw")).unwrap(); + let overlay = fs::read(pool.data().join("overlay.raw")).unwrap(); + let history = pool.directory.join("historical-graph.json"); + fs::write(&history, b"historical-source-binding").unwrap(); + let inspected = pool.selected(); + pool.unchanged(&old); + let result = pool.apply(&inspected.selection_sha256).unwrap(); + assert_eq!(result, inspected); + let mut expected = pool.owner.clone(); + expected.storage.as_mut().unwrap().device = inspected.new_device; + expected.overlay.as_mut().unwrap().device = inspected.new_device; + expected.project_share.as_mut().unwrap().device = inspected.new_device; + assert_eq!( + Owner::load_for_short_home_recovery(&pool.candidate).unwrap(), + expected + ); + assert!(pool.owner.short_home.symlink_metadata().is_err()); + assert_eq!(fs::read(history).unwrap(), b"historical-source-binding"); + assert_eq!(fs::read(pool.data().join("storage.raw")).unwrap(), storage); + assert_eq!(fs::read(pool.data().join("overlay.raw")).unwrap(), overlay); + assert!(pool.apply(&inspected.selection_sha256).is_err()); + // Ordinary recovery still owns HOME restoration and the recovered phase. + let recovered = crate::provider::recover(&pool.candidate).unwrap(); + assert_eq!(recovered.phase, "recovered-unclean"); + assert_eq!(recovered.process_alive, Some(false)); + assert_eq!(fs::read(pool.data().join("storage.raw")).unwrap(), storage); +} + +#[test] +fn disk_and_share_changes_beyond_common_device_refuse_without_writes() { + for case in 0..7 { + let mut pool = Pool::new(); + let observed_device = identity::disk(&pool.data().join("storage.raw")) + .unwrap() + .device; + let disk = pool.owner.storage.as_mut().unwrap(); + match case { + 0 => disk.inode += 1, + 1 => disk.bytes += 1, + 2 => disk.uuid.push('0'), + 3 => disk.device = observed_device, + 4 => pool.owner.overlay.as_mut().unwrap().device += 1, + 5 => pool.owner.project_share.as_mut().unwrap().inode += 1, + _ => pool.owner.project_share.as_mut().unwrap().device += 1, + } + pool.owner.save(&pool.candidate).unwrap(); + let before = pool.receipt(); + assert!(pool.apply(&"a".repeat(64)).is_err()); + pool.unchanged(&before); + } +} + +#[test] +fn stale_owner_or_host_boot_selection_refuses() { + let mut pool = Pool::new(); + let selection = pool.selected(); + assert!(pool.apply(&"a".repeat(64)).is_err()); + pool.owner.phase = "stopped".into(); + pool.owner.save(&pool.candidate).unwrap(); + let before = pool.receipt(); + assert!(pool.apply(&selection.selection_sha256).is_err()); + pool.unchanged(&before); + let selected = pool.selected(); + assert!( + recover_with_boot(&pool.candidate, &selected.selection_sha256, || Ok(pool + .boot + + 1)) + .is_err() + ); + pool.unchanged(&before); +} + +#[test] +fn same_boot_live_reused_pid_and_unknown_identity_refuse() { + for case in 0..4 { + let mut pool = Pool::new(); + if case == 0 { + pool.boot = pool.owner.process.as_ref().unwrap().start_micros; + } + if case == 1 || case == 2 { + let p = pool.owner.process.as_mut().unwrap(); + p.pid = std::process::id() as i32; + if case == 1 { + p.start_micros = identity::observe(p.pid).unwrap().start_micros; + } + } + if case == 3 { + pool.owner.process = None; + } + pool.owner.save(&pool.candidate).unwrap(); + let before = pool.receipt(); + assert!(pool.apply(&"a".repeat(64)).is_err()); + pool.unchanged(&before); + } +} + +#[test] +fn held_operation_vm_lock_and_disk_handles_refuse() { + for case in 0..3 { + let pool = Pool::new(); + let selected = pool.selected(); + let before = pool.receipt(); + let _operation = + (case == 0).then(|| state::Lock::acquire_existing(&root(&pool.candidate)).unwrap()); + let file = if case == 1 { + Some(File::open(pool.data().join("vm.lock")).unwrap()) + } else if case == 2 { + Some(File::open(pool.data().join("storage.raw")).unwrap()) + } else { + None + }; + if case == 1 { + assert_eq!( + unsafe { + libc::flock( + file.as_ref().unwrap().as_raw_fd(), + libc::LOCK_EX | libc::LOCK_NB, + ) + }, + 0 + ); + } + assert!(pool.apply(&selected.selection_sha256).is_err()); + pool.unchanged(&before); + } +} + +#[test] +fn pending_foreign_updates_and_prepared_pools_are_preserved() { + for name in [ + "owner.pending", + "network-update.json", + "network-update.pending", + "prepared-base.json", + "prepared-base.json.pending", + ] { + let pool = Pool::new(); + let selected = pool.selected(); + let before = pool.receipt(); + let path = root(&pool.candidate).join(name); + fs::write(&path, b"foreign-or-interrupted").unwrap(); + assert!(pool.apply(&selected.selection_sha256).is_err()); + pool.unchanged(&before); + assert_eq!(fs::read(path).unwrap(), b"foreign-or-interrupted"); + } +} + +#[test] +fn foreign_alias_and_source_substitution_refuse() { + for case in 0..2 { + let pool = Pool::new(); + let selected = pool.selected(); + let before = pool.receipt(); + if case == 0 { + fs::write(&pool.owner.short_home, b"foreign").unwrap(); + } else { + let project = &pool.owner.project_share.as_ref().unwrap().project; + fs::rename(project, pool.directory.join("preserved-app")).unwrap(); + state::private_directory(project).unwrap(); + fs::write(project.join("package.json"), b"{}").unwrap(); + } + assert!(pool.apply(&selected.selection_sha256).is_err()); + assert_eq!(pool.receipt(), before); + if case == 0 { + assert_eq!(fs::read(&pool.owner.short_home).unwrap(), b"foreign"); + } + } +} + +#[test] +fn final_recheck_rejects_identity_change_before_publication() { + let pool = Pool::new(); + let selected = pool.selected(); + let before = pool.receipt(); + let called = std::cell::Cell::new(false); + let result = recover_with_boot(&pool.candidate, &selected.selection_sha256, || { + if called.replace(true) { + fs::write(pool.data().join("storage.raw"), b"changed").unwrap(); + } + Ok(pool.boot) + }); + assert!(result.is_err()); + pool.unchanged(&before); + assert_eq!( + fs::read(pool.data().join("storage.raw")).unwrap(), + b"changed" + ); +} diff --git a/packages/runtime-core/src/provider/mod.rs b/packages/runtime-core/src/provider/mod.rs index 30f7082bf..e851f1890 100644 --- a/packages/runtime-core/src/provider/mod.rs +++ b/packages/runtime-core/src/provider/mod.rs @@ -45,6 +45,7 @@ pub mod resources; pub mod storage_usage; pub use image_load::load as load_image; mod lifecycle; +pub use lifecycle::host_filesystem; mod network_intent; mod network_update; pub use network_update::{enable_internet, extend_network}; diff --git a/packages/runtime-core/src/provider/state.rs b/packages/runtime-core/src/provider/state.rs index e8810dbe1..bc4c3bf84 100644 --- a/packages/runtime-core/src/provider/state.rs +++ b/packages/runtime-core/src/provider/state.rs @@ -176,7 +176,7 @@ impl Default for ReclamationPolicy { } } -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Owner { #[serde(default, skip_serializing_if = "Option::is_none")] From 4b5bdff9ac6cc97d3d13970aac6ce5a6c59b0435 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Tue, 29 Sep 2026 22:16:52 -0400 Subject: [PATCH 03/25] fix: fence replaced lock paths before filesystem recovery --- .../src/provider/lifecycle/host_filesystem.rs | 40 ++++++++++++++++--- .../lifecycle/host_filesystem/tests.rs | 25 ++++++++++++ 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs b/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs index 08467b8f3..6c9e10634 100644 --- a/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs +++ b/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs @@ -8,7 +8,7 @@ use super::{Owner, binary, identity, lock_absent_disks, root, state}; use crate::{Candidate, CandidateError}; use serde::Serialize; use sha2::{Digest, Sha256}; -use std::{fs, os::unix::fs::MetadataExt}; +use std::{fs, os::unix::fs::MetadataExt, path::Path}; #[derive(Debug, Serialize, PartialEq, Eq)] pub struct Inspection { @@ -121,6 +121,34 @@ fn same_disk(before: &identity::DiskIdentity, after: &identity::DiskIdentity) -> && before.uuid == after.uuid } +/// A retained flock cannot fence a writer that opens a substituted lock pathname. +fn bound_locks( + candidate: &Candidate, + owner: &Owner, + operation: &state::Lock, + vm: &fs::File, +) -> Result<(), CandidateError> { + let check = |path: &Path, expected: (u64, u64)| -> Result<(), CandidateError> { + let metadata = fs::symlink_metadata(path).map_err(state::io)?; + if !metadata.is_file() + || metadata.nlink() != 1 + || (metadata.dev(), metadata.ino()) != expected + { + return Err(refused("Held lock pathname was replaced")); + } + Ok(()) + }; + check( + &root(candidate).join("operation.lock"), + operation.identity()?, + )?; + let metadata = vm.metadata().map_err(state::io)?; + check( + &owner.real_data_dir(candidate)?.join("vm.lock"), + (metadata.dev(), metadata.ino()), + ) +} + /// Build a candidate owner in memory. This function neither adopts a new disk nor writes state. fn selection( candidate: &Candidate, @@ -204,13 +232,14 @@ fn selection( /// Read-only inspection holds both existing locks and proves disks have no open handles. pub fn inspect(candidate: &Candidate) -> Result { - let _operation = state::Lock::acquire_existing(&root(candidate))?; + let operation = state::Lock::acquire_existing(&root(candidate))?; let owner = Owner::load_for_short_home_recovery(candidate)?; let (inspection, next) = selection(candidate, &owner, host_boot_micros()?)?; - let _vm = lock_absent_disks(candidate, &next)?; + let vm = lock_absent_disks(candidate, &next)?; if selection(candidate, &owner, host_boot_micros()?)?.0 != inspection { return Err(refused("Inspection changed during absence verification")); } + bound_locks(candidate, &owner, &operation, &vm)?; Ok(inspection) } @@ -229,16 +258,17 @@ fn recover_with_boot( if expected.len() != 64 || !expected.bytes().all(|b| b.is_ascii_hexdigit()) { return Err(refused("An exact inspection SHA-256 is required")); } - let _operation = state::Lock::acquire_existing(&root(candidate))?; + let operation = state::Lock::acquire_existing(&root(candidate))?; let owner = Owner::load_for_short_home_recovery(candidate)?; let (inspection, next) = selection(candidate, &owner, boot()?)?; if inspection.selection_sha256 != expected { return Err(refused("Inspection selection is stale")); } - let _vm = lock_absent_disks(candidate, &next)?; + let vm = lock_absent_disks(candidate, &next)?; if selection(candidate, &owner, boot()?)?.0 != inspection { return Err(refused("Selected identities changed before publication")); } + bound_locks(candidate, &owner, &operation, &vm)?; next.save(candidate)?; if Owner::load_for_short_home_recovery(candidate)? != next { return Err(CandidateError::new( diff --git a/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs b/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs index 3585833bf..4c6cc502f 100644 --- a/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs +++ b/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs @@ -292,3 +292,28 @@ fn final_recheck_rejects_identity_change_before_publication() { b"changed" ); } + +#[test] +fn substituted_lock_paths_do_not_authorize_publication_on_an_old_descriptor() { + for name in ["operation.lock", "vm.lock"] { + let pool = Pool::new(); + let selected = pool.selected(); + let before = pool.receipt(); + let called = std::cell::Cell::new(false); + let path = if name == "operation.lock" { + root(&pool.candidate).join(name) + } else { + pool.data().join(name) + }; + let result = recover_with_boot(&pool.candidate, &selected.selection_sha256, || { + if called.replace(true) { + fs::rename(&path, path.with_extension("preserved")).unwrap(); + fs::write(&path, b"replacement lock").unwrap(); + } + Ok(pool.boot) + }); + assert!(matches!(result, Err(e) if e.code == "host_filesystem_recovery")); + pool.unchanged(&before); + assert_eq!(fs::read(path).unwrap(), b"replacement lock"); + } +} From 3266931063bf33339966bcff4f6ee7d497387631 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 01:45:05 -0400 Subject: [PATCH 04/25] fix: explicitly recover retained graph host pins after reboot --- docs/guides/native-candidate.md | 35 + packages/runtime-core/src/graph_cli.rs | 44 ++ packages/runtime-core/src/main.rs | 2 + .../src/provider/graph/bridges/cleanup.rs | 78 +- .../src/provider/graph/dead_owner_cleanup.rs | 194 ++++- .../src/provider/graph/dependency_slots.rs | 165 +++- .../src/provider/graph/foreground.rs | 1 - .../provider/graph/foreground/native_test.rs | 1 + .../native_test/host_pin_recovery.rs | 537 +++++++++++++ .../provider/graph/foreground/transport.rs | 202 ++++- .../graph/foreground/transport_tests.rs | 72 ++ .../src/provider/graph/host_pin_recovery.rs | 704 ++++++++++++++++++ .../src/provider/graph/host_relay.rs | 23 +- .../src/provider/graph/live_owner_cleanup.rs | 2 +- .../runtime-core/src/provider/graph/mod.rs | 4 + .../runtime-core/src/provider/host_pin.rs | 40 + .../src/provider/lifecycle/host_filesystem.rs | 4 +- packages/runtime-core/src/provider/mod.rs | 2 + .../src/provider/relay_owner/publication.rs | 62 +- .../provider/relay_owner/publication/dead.rs | 1 + 20 files changed, 2099 insertions(+), 74 deletions(-) create mode 100644 packages/runtime-core/src/provider/graph/foreground/native_test/host_pin_recovery.rs create mode 100644 packages/runtime-core/src/provider/graph/host_pin_recovery.rs create mode 100644 packages/runtime-core/src/provider/host_pin.rs diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 752281131..010e75fff 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -165,6 +165,41 @@ graph receipts. Use ordinary recovery afterward. Historical shared-source graphs retain the prior identity and may require verified cleanup plus a new generation; successful metadata migration alone does not establish application recovery. +After that explicit provider recovery and one audited VM boot, a retained graph +from the immediately preceding guest boot can still carry old host device numbers +in its publisher, relay-control, and dependency-socket receipts. Inspect and select +one run's host-pin recovery separately: + +```sh +./hack-native --candidate-root /absolute/private/candidate-home graph inspect-host-pin-recovery --run-id --json +./hack-native --candidate-root /absolute/private/candidate-home graph recover-host-pins --run-id --expect-selection --accept-legacy-device-rebind --json +./hack-native --candidate-root /absolute/private/candidate-home graph recover-cleanup --run-id --expect-receipt --json +./hack-native --candidate-root /absolute/private/candidate-home graph retire-recovered-publisher --run-id --expect-owner --json +``` + +The first command only inspects. The second publishes a private, exact-run +witness before cleanup; it changes no graph, publisher, control, or dependency +receipt. It requires the provider's repaired current disks/source, unchanged +recorded inodes and raw receipts, dead owners whose recorded starts precede the +current physical host boot, refused socket listeners, and the immediate guest +boot transition. It also binds retained volume names and actual labels. Cleanup +and retirement can use the witness only for those selected old pins; ordinary +reads and later publisher generations remain strict. Missing or foreign pins, +active listeners, changed resources, pending journals, or further guest boots +refuse without adopting another run. A completed older cleanup journal is +retained as history and does not itself block a later selected generation. + +Legacy receipts do not identify the original APFS volume. This explicit +device-number rebind cannot prove pre-reboot volume continuity. Completing these +commands retains the old run's data and proves cleanup of its dead generation; +it does not migrate `graph.source.shared`, restore the application, establish +route readiness, or claim overall v5 acceptance. Same-run source continuity +requires a separate explicit witness-bound transition after cleanup and +publisher retirement. If macOS removed the foreground or relay-control +directory itself during reboot, this command refuses: the old pin receipts no +longer exist, and absent pathnames cannot stand in for their recorded owner +identities. That case requires a separate selected absence-recovery procedure. + Compatibility is qualified for specific bundle hashes and state formats. The displayed version alone does not establish frontend/executor or downgrade compatibility. V4 and candidate homes remain separate; this procedure does not diff --git a/packages/runtime-core/src/graph_cli.rs b/packages/runtime-core/src/graph_cli.rs index 9b9e3b52c..b93a6872a 100644 --- a/packages/runtime-core/src/graph_cli.rs +++ b/packages/runtime-core/src/graph_cli.rs @@ -25,6 +25,50 @@ pub fn command(candidate: &Candidate, args: &[&str]) -> Result run, + _ => return Err(invalid()), + }; + #[cfg(target_os = "macos")] + { + return graph::inspect_host_pin_recovery(candidate, run); + } + #[cfg(not(target_os = "macos"))] + { + let _ = run; + return Err(invalid()); + } + } + if *action == "recover-host-pins" { + let (run, expected) = match *args { + [ + "--run-id", + run, + "--expect-selection", + expected, + "--accept-legacy-device-rebind", + ] + | [ + "--run-id", + run, + "--expect-selection", + expected, + "--accept-legacy-device-rebind", + "--json", + ] => (run, expected), + _ => return Err(invalid()), + }; + #[cfg(target_os = "macos")] + { + return graph::recover_host_pins(candidate, run, expected); + } + #[cfg(not(target_os = "macos"))] + { + let _ = (run, expected); + return Err(invalid()); + } + } if ["run-selection", "run-service"].contains(action) { return one_off::command(candidate, action, args); } diff --git a/packages/runtime-core/src/main.rs b/packages/runtime-core/src/main.rs index be6ee57a7..5878fae07 100644 --- a/packages/runtime-core/src/main.rs +++ b/packages/runtime-core/src/main.rs @@ -27,6 +27,8 @@ Usage: hack-local graph recover-cleanup --run-id <32-hex> --expect-receipt [--json] hack-local graph recover-live-owner --run-id <32-hex> --expect-receipt [--json] hack-local graph retire-recovered-publisher --run-id <32-hex> --expect-owner <32-hex> [--json] + hack-local graph inspect-host-pin-recovery --run-id <32-hex> [--json] + hack-local graph recover-host-pins --run-id <32-hex> --expect-selection <64-hex> --accept-legacy-device-rebind [--json] hack-local graph logs --run-id <32-hex> --service [--tail <1..1000>] [--json] hack-local graph exec --run-id <32-hex> --service [--workdir /path] [--timeout-seconds <1..120>] [--json] -- [args...] hack-local graph dependency-plan --dependencies [--json] diff --git a/packages/runtime-core/src/provider/graph/bridges/cleanup.rs b/packages/runtime-core/src/provider/graph/bridges/cleanup.rs index 8d918fe5f..c76005ce1 100644 --- a/packages/runtime-core/src/provider/graph/bridges/cleanup.rs +++ b/packages/runtime-core/src/provider/graph/bridges/cleanup.rs @@ -46,6 +46,14 @@ fn validate_selection( engine: &Engine<'_>, receipt: &Receipt, selection: &Selection, +) -> Result<(), CandidateError> { + validate_selection_recovery(engine, receipt, selection, None) +} +fn validate_selection_recovery( + engine: &Engine<'_>, + receipt: &Receipt, + selection: &Selection, + host_pin: Option<&super::super::host_pin_recovery::Witness>, ) -> Result<(), CandidateError> { let capacity = engine.guest().bridge_intent().map_or(0, |v| v.slots); if selection.version != 1 @@ -67,6 +75,7 @@ fn validate_selection( if predecessor_owner( receipt, selection.previous_boot.as_deref().ok_or_else(invalid)?, + host_pin, )? != *expected { return Err(invalid()); @@ -183,17 +192,39 @@ fn pinned_predecessor_eligible(receipt: &Receipt) -> bool { }) } -fn predecessor_owner(receipt: &Receipt, previous: &str) -> Result { +fn predecessor_owner( + receipt: &Receipt, + previous: &str, + host_pin: Option<&super::super::host_pin_recovery::Witness>, +) -> Result { let startup = receipt .relay_startup .as_ref() .filter(|_| pinned_predecessor_eligible(receipt)) .ok_or_else(invalid)?; - let pin = crate::provider::relay_owner::publication::PinnedEndpoint::load( - &startup.control_root, - super::super::host_relay::context(&receipt.owner, previous)?, - )?; - pin.verify_dead()?; + let context = super::super::host_relay::context(&receipt.owner, previous)?; + let pin = if let Some(selected) = host_pin { + if !selected.matches_graph(receipt) || selected.control_root() != startup.control_root { + return Err(invalid()); + } + let pin = crate::provider::relay_owner::publication::PinnedEndpoint::load_legacy_recovery( + &startup.control_root, + context, + selected.rebind(), + selected.host_boot_micros(), + )?; + if &pin.legacy_summary() != selected.control() { + return Err(invalid()); + } + pin + } else { + let pin = crate::provider::relay_owner::publication::PinnedEndpoint::load( + &startup.control_root, + context, + )?; + pin.verify_dead()?; + pin + }; Ok(pin .fingerprint() .iter() @@ -208,6 +239,7 @@ pub(crate) fn capture_previous_boot( engine: &Engine<'_>, receipt: &Receipt, previous: &str, + host_pin: Option<&super::super::host_pin_recovery::Witness>, ) -> Result { let store = strict_store(candidate, engine)?; let mut selection = Selection { @@ -271,22 +303,23 @@ pub(crate) fn capture_previous_boot( }) .collect(); } else { - selection.predecessor_owner = Some(predecessor_owner(receipt, previous)?); + selection.predecessor_owner = Some(predecessor_owner(receipt, previous, host_pin)?); } } - validate_selection(engine, receipt, &selection)?; + validate_selection_recovery(engine, receipt, &selection, host_pin)?; Ok(selection) } /// Pending recovery may resume only the exact remaining reservations. Released /// slots may be absent; a replacement or new reservation is never adopted. -pub(crate) fn verify_remaining( +pub(crate) fn verify_remaining_recovery( candidate: &Candidate, engine: &Engine<'_>, receipt: &Receipt, selection: &Selection, + host_pin: Option<&super::super::host_pin_recovery::Witness>, ) -> Result<(), CandidateError> { - validate_selection(engine, receipt, selection)?; + validate_selection_recovery(engine, receipt, selection, host_pin)?; if selection.previous_boot.is_none() { return Ok(()); } @@ -529,19 +562,28 @@ pub(crate) fn read( engine: &Engine<'_>, receipt: &Receipt, root: &std::path::Path, +) -> Result { + read_recovery(engine, receipt, root, None) +} +pub(crate) fn read_recovery( + engine: &Engine<'_>, + receipt: &Receipt, + root: &std::path::Path, + host_pin: Option<&super::super::host_pin_recovery::Witness>, ) -> Result { let selection = state::read_bounded(&selection_path(root)?, 65536)?; - validate_selection(engine, receipt, &selection)?; + validate_selection_recovery(engine, receipt, &selection, host_pin)?; Ok(selection) } -pub(crate) fn verify( +pub(crate) fn verify_recovery( candidate: &Candidate, engine: &Engine<'_>, receipt: &Receipt, selection: &Selection, + host_pin: Option<&super::super::host_pin_recovery::Witness>, ) -> Result<(), CandidateError> { - validate_selection(engine, receipt, selection)?; + validate_selection_recovery(engine, receipt, selection, host_pin)?; let store = strict_store(candidate, engine)?; if store.next_launch_serial < selection.serial || store.slots.values().any(|a| a.run == receipt.run) @@ -557,6 +599,16 @@ pub(crate) fn verify( Ok(()) } +#[cfg(test)] +pub(crate) fn verify( + candidate: &Candidate, + engine: &Engine<'_>, + receipt: &Receipt, + selection: &Selection, +) -> Result<(), CandidateError> { + verify_recovery(candidate, engine, receipt, selection, None) +} + #[cfg(test)] mod tests { use super::*; diff --git a/packages/runtime-core/src/provider/graph/dead_owner_cleanup.rs b/packages/runtime-core/src/provider/graph/dead_owner_cleanup.rs index f2d1a8135..9e32a865f 100644 --- a/packages/runtime-core/src/provider/graph/dead_owner_cleanup.rs +++ b/packages/runtime-core/src/provider/graph/dead_owner_cleanup.rs @@ -4,6 +4,7 @@ use super::*; use crate::provider::{identity, lifecycle, state::Owner}; use sha2::{Digest, Sha256}; +use std::os::unix::fs::MetadataExt; const FILE: &str = "dead-owner-cleanup.json"; #[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -322,18 +323,98 @@ pub fn retire_recovered_publisher( } let bridges = intent.bridges.as_ref().ok_or_else(refused)?; bridges::cleanup::verify_recovery_file(&root, bridges, intent.prior_bridges.as_ref())?; + let legacy = super::host_pin_recovery::load_witness(candidate, run)?.filter(|witness| { + witness.graph_sha256() == intent.original_sha256 + && witness.publisher_sha256() == intent.owner_sha256 + && witness.matches_graph(&intent.original) + }); if intent.new_boot.as_deref() == Some(engine.guest().boot_id()) { - host_relay::inspect_cleanup(candidate, &engine, &receipt, false, &environment, bridges)?; - foreground::retire_publisher_path(candidate, run, &intent.owner_sha256, &complete)?; + if let Some(witness) = legacy { + if let Some(retired) = foreground::transport::Retired::acquire(candidate, run)? { + let control_root = witness.control_root().join("relay-control"); + let control_lock = state::Lock::acquire_existing(&control_root)?; + host_relay::inspect_cleanup_recovery( + candidate, + &engine, + &receipt, + false, + &environment, + bridges, + Some(&witness), + )?; + let lock_path = fs::symlink_metadata(control_root.join("operation.lock")) + .map_err(|_| refused())?; + if (lock_path.dev(), lock_path.ino()) != control_lock.identity()? { + return Err(refused()); + } + retired.verify_recovery_with_rebind( + candidate, + run, + &intent.owner_sha256, + &complete, + Some(witness.rebind()), + )?; + } else { + let foreground_root = foreground::transport::root(candidate, run)?; + let foreground_lock = state::Lock::acquire_existing(&foreground_root)?; + let guard = super::host_pin_recovery::acquire_for_cleanup( + candidate, + run, + &engine, + super::host_pin_recovery::CleanupProof { + original: &intent.original, + sha256: &intent.original_sha256, + current_is_original: false, + allow_absent_reservation: true, + publisher_may_be_partial: true, + }, + witness, + )?; + host_relay::inspect_cleanup_recovery( + candidate, + &engine, + &receipt, + false, + &environment, + bridges, + Some(guard.witness()), + )?; + guard.verify_lock()?; + let lock_path = fs::symlink_metadata(foreground_root.join("operation.lock")) + .map_err(|_| refused())?; + if (lock_path.dev(), lock_path.ino()) != foreground_lock.identity()? { + return Err(refused()); + } + foreground::transport::retire_recovered_publisher_locked( + candidate, + run, + &intent.owner_sha256, + &complete, + Some(guard.witness().rebind()), + &foreground_lock, + )?; + } + } else { + host_relay::inspect_cleanup( + candidate, + &engine, + &receipt, + false, + &environment, + bridges, + )?; + foreground::retire_publisher_path(candidate, run, &intent.owner_sha256, &complete)?; + } } else { // A later VM boot has a new bridge registry generation. Recheck the // retained graph and immutable prior proof, then require the publisher // to have been fully retired under the original recovery boot. - foreground::verify_recovered_publisher_retired( + foreground::transport::verify_recovered_publisher_retired_recovery( candidate, run, &intent.owner_sha256, &complete, + legacy.as_ref().map(|witness| witness.rebind()), )?; } Ok(json!({"run":run,"publisher_retired":true,"data_retained":true})) @@ -427,7 +508,13 @@ fn fresh_boot( } fn execute(candidate: &Candidate, run: &str, expected: &str) -> Result { let engine = Engine::connect_cleanup_wait(candidate)?; - let dead = foreground::DeadOwner::acquire(candidate, run)?; + let selected_pin = super::host_pin_recovery::selected_for_old_publisher(candidate, run)?; + let dead = foreground::DeadOwner::acquire_recovery( + candidate, + run, + selected_pin.as_ref().map(|pin| pin.rebind()), + selected_pin.as_ref().map(|pin| pin.host_boot_micros()), + )?; let (receipt, root) = load(candidate, &engine, run)?; if exists(&root.join("one-off-normalization.json"))? { let intent: Intent = state::read(&root.join(FILE))?; @@ -448,6 +535,36 @@ fn execute(candidate: &Candidate, run: &str, expected: &str) -> Result = if exists(&root.join(FILE))? { + Some(state::read(&root.join(FILE))?) + } else { + None + }; + let pin_guard = if let Some(witness) = selected_pin { + let original = existing_intent + .as_ref() + .map_or(&receipt, |intent| &intent.original); + let original_sha = existing_intent + .as_ref() + .map_or(expected, |intent| intent.original_sha256.as_str()); + Some(super::host_pin_recovery::acquire_for_cleanup( + candidate, + run, + &engine, + super::host_pin_recovery::CleanupProof { + original, + sha256: original_sha, + current_is_original: existing_intent.is_none(), + allow_absent_reservation: existing_intent + .as_ref() + .is_some_and(|intent| intent.complete_sha256.is_some()), + publisher_may_be_partial: false, + }, + witness, + )?) + } else { + None + }; let mut intent: Intent = if exists(&root.join(FILE))? { state::read(&root.join(FILE))? } else { @@ -479,8 +596,13 @@ fn execute(candidate: &Candidate, run: &str, expected: &str) -> Result Result Result Result Result Result Result Result Result { } } fn read(path: &Path) -> Result { + read_with_bytes(path).map(|(record, _)| record) +} +fn read_with_bytes(path: &Path) -> Result<(Record, Vec), CandidateError> { let mut file = OpenOptions::new() .read(true) .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) @@ -70,7 +74,121 @@ fn read(path: &Path) -> Result { if bytes.len() as u64 != m.len() { return Err(refused()); } - serde_json::from_slice(&bytes).map_err(|_| refused()) + let record = serde_json::from_slice(&bytes).map_err(|_| refused())?; + Ok((record, bytes)) +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct LegacyReservation { + pub record_sha256: String, + pub sockets: BTreeMap, + pub process: ProcessIdentity, +} +pub(super) fn inspect_legacy( + candidate: &Candidate, + run: &str, + rebind: DeviceRebind, + host_boot_micros: u64, +) -> Result, CandidateError> { + let owner = state::Owner::load(candidate)?; + let directory = root(candidate); + let values = records( + &directory, + &owner.token, + owner.dependency_sockets.map_or(0, |v| v.slots), + )?; + let Some(record) = values.iter().find(|r| r.run == run) else { + return Ok(None); + }; + let (again, bytes) = read_with_bytes(&directory.join(format!("{run}.json")))?; + if &again != record || record.sockets.len() != record.slots.len() { + return Err(refused()); + } + // SAFETY: geteuid has no arguments or side effects. + identity::verify( + &record.process, + &record.process, + &record.process.executable, + unsafe { libc::geteuid() }, + )?; + rebind.definitely_dead_before_boot(&record.process, host_boot_micros)?; + for slot in record.slots.values() { + let expected = record.sockets.get(slot).ok_or_else(refused)?; + let path = socket(&owner.short_home, *slot); + let metadata = fs::symlink_metadata(&path).map_err(|_| refused())?; + if !metadata.file_type().is_socket() + || metadata.uid() != record.process.uid + || metadata.mode() & 0o7777 != 0o600 + || metadata.nlink() != 1 + || !rebind.matches(*expected, (metadata.dev(), metadata.ino())) + { + return Err(refused()); + } + crate::provider::relay_owner::publication::dead::no_listener(&path) + .map_err(|_| refused())?; + } + Ok(Some(LegacyReservation { + record_sha256: format!("{:x}", Sha256::digest(bytes)), + sockets: record.sockets.clone(), + process: record.process.clone(), + })) +} +pub(super) fn verify_legacy_remaining( + candidate: &Candidate, + run: &str, + rebind: DeviceRebind, + selected: &LegacyReservation, + allow_absent_record: bool, +) -> Result<(), CandidateError> { + let owner = state::Owner::load(candidate)?; + let path = root(candidate).join(format!("{run}.json")); + if absent(&path)? { + if !allow_absent_record { + return Err(refused()); + } + for slot in selected.sockets.keys() { + if !absent(&socket(&owner.short_home, *slot))? { + return Err(refused()); + } + } + return Ok(()); + } + let values = records( + &root(candidate), + &owner.token, + owner.dependency_sockets.map_or(0, |v| v.slots), + )?; + let record = values + .iter() + .find(|record| record.run == run) + .ok_or_else(refused)?; + let (same, bytes) = read_with_bytes(&path)?; + if &same != record + || selected.record_sha256 != format!("{:x}", Sha256::digest(bytes)) + || selected.sockets != record.sockets + || selected.process != record.process + { + return Err(refused()); + } + for slot in record.slots.values() { + let path = socket(&owner.short_home, *slot); + if absent(&path)? { + continue; + } + let expected = record.sockets.get(slot).ok_or_else(refused)?; + let observed = fs::symlink_metadata(&path).map_err(|_| refused())?; + if !observed.file_type().is_socket() + || observed.uid() != record.process.uid + || observed.mode() & 0o7777 != 0o600 + || observed.nlink() != 1 + || !rebind.matches(*expected, (observed.dev(), observed.ino())) + { + return Err(refused()); + } + crate::provider::relay_owner::publication::dead::no_listener(&path) + .map_err(|_| refused())?; + } + Ok(()) } fn valid(record: &Record, owner: &str, capacity: u8) -> bool { record.version == 1 @@ -339,7 +457,11 @@ pub fn inspect(candidate: &Candidate) -> Result Result<(), CandidateError> { +fn recover_record( + candidate: &Candidate, + record: &Record, + rebind: Option, +) -> Result<(), CandidateError> { if identity::alive(record.process.pid)? { return Err(refused()); } @@ -347,10 +469,14 @@ fn recover_record(candidate: &Candidate, record: &Record) -> Result<(), Candidat if record.owner != owner.token { return Err(refused()); } - recover_paths(&owner.short_home, record)?; + recover_paths(&owner.short_home, record, rebind)?; remove_record(&root(candidate), record) } -fn recover_paths(home: &Path, record: &Record) -> Result<(), CandidateError> { +fn recover_paths( + home: &Path, + record: &Record, + rebind: Option, +) -> Result<(), CandidateError> { // Validate every selected path before removing any. Unexpected or unrecorded // paths are never adopted, even when nobody currently listens on them. let mut selected = Vec::new(); @@ -364,7 +490,11 @@ fn recover_paths(home: &Path, record: &Record) -> Result<(), CandidateError> { || m.mode() & 0o7777 != 0o600 || m.uid() != record.process.uid || m.nlink() != 1 - || record.sockets.get(slot) != Some(&(m.dev(), m.ino())) + || !record.sockets.get(slot).is_some_and(|expected| { + rebind.map_or(*expected == (m.dev(), m.ino()), |v| { + v.matches(*expected, (m.dev(), m.ino())) + }) + }) { return Err(refused()); } @@ -402,13 +532,14 @@ pub fn recover_orphan( if fingerprint(record)? != expected || !absent(&super::directory(candidate, run)?)? { return Err(refused()); } - recover_record(candidate, record)?; + recover_record(candidate, record, None)?; Ok(serde_json::json!({"run":run,"released":true})) } /// Caller holds Engine lease and has completed dead-owner cleanup/verification. pub(super) fn recover_cleaned( candidate: &Candidate, receipt: &Receipt, + rebind: Option<(DeviceRebind, &LegacyReservation)>, ) -> Result<(), CandidateError> { let directory = root(candidate); if absent(&directory.join(format!("{}.json", receipt.run)))? { @@ -438,7 +569,17 @@ pub(super) fn recover_cleaned( { return Err(refused()); } - recover_record(candidate, record) + if let Some((_, selected)) = rebind { + let (again, bytes) = read_with_bytes(&directory.join(format!("{}.json", receipt.run)))?; + if &again != record + || selected.record_sha256 != format!("{:x}", Sha256::digest(bytes)) + || selected.sockets != record.sockets + || selected.process != record.process + { + return Err(refused()); + } + } + recover_record(candidate, record, rebind.map(|(value, _)| value)) } #[cfg(test)] @@ -555,14 +696,14 @@ mod tests { let mut b = fixture_record('2', 1); let a_listener = listener(&fixture.0, &mut a, 0); let b_listener = listener(&fixture.0, &mut b, 1); - assert!(recover_paths(&fixture.0, &a).is_err()); + assert!(recover_paths(&fixture.0, &a, None).is_err()); drop(a_listener); let old = a.sockets[&0]; a.sockets.insert(0, (old.0, old.1 + 1)); - assert!(recover_paths(&fixture.0, &a).is_err()); + assert!(recover_paths(&fixture.0, &a, None).is_err()); assert!(socket(&fixture.0, 0).exists()); a.sockets.insert(0, old); - recover_paths(&fixture.0, &a).unwrap(); + recover_paths(&fixture.0, &a, None).unwrap(); assert!(!socket(&fixture.0, 0).exists()); assert!(UnixStream::connect(socket(&fixture.0, 1)).is_ok()); drop(b_listener); @@ -575,10 +716,10 @@ mod tests { let first = listener(&fixture.0, &mut a, 0); let second = listener(&fixture.0, &mut a, 1); drop(first); - assert!(recover_paths(&fixture.0, &a).is_err()); + assert!(recover_paths(&fixture.0, &a, None).is_err()); assert!(socket(&fixture.0, 0).exists()); drop(second); - recover_paths(&fixture.0, &a).unwrap(); + recover_paths(&fixture.0, &a, None).unwrap(); assert!(!socket(&fixture.0, 0).exists()); assert!(!socket(&fixture.0, 1).exists()); } diff --git a/packages/runtime-core/src/provider/graph/foreground.rs b/packages/runtime-core/src/provider/graph/foreground.rs index 7f46e84be..366769dc6 100644 --- a/packages/runtime-core/src/provider/graph/foreground.rs +++ b/packages/runtime-core/src/provider/graph/foreground.rs @@ -20,7 +20,6 @@ mod tests; pub(in crate::provider::graph) mod transport; pub(in crate::provider::graph) use transport::DeadOwner; pub(in crate::provider::graph) use transport::retire_recovered_publisher as retire_publisher_path; -pub(in crate::provider::graph) use transport::verify_recovered_publisher_retired; use transport::{Publication, WireRequest}; fn refused() -> CandidateError { CandidateError::new( diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test.rs b/packages/runtime-core/src/provider/graph/foreground/native_test.rs index 016db1b0f..aba7ac312 100644 --- a/packages/runtime-core/src/provider/graph/foreground/native_test.rs +++ b/packages/runtime-core/src/provider/graph/foreground/native_test.rs @@ -542,5 +542,6 @@ mod dependency_rebind; mod dependency_slots; mod startup_cancellation; +mod host_pin_recovery; mod retired_recovery_cleanup; mod same_boot_recovery; diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test/host_pin_recovery.rs b/packages/runtime-core/src/provider/graph/foreground/native_test/host_pin_recovery.rs new file mode 100644 index 000000000..1ce87e513 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/foreground/native_test/host_pin_recovery.rs @@ -0,0 +1,537 @@ +//! Real CLI path for explicitly witnessed legacy host-pin cleanup. An isolated +//! test VM supplies the guest; only fixture receipt device numbers are changed. +use super::dependency_rebind::{checked_cli, exec, snapshot}; +use super::*; +use std::{ + io::Write, + os::unix::fs::{MetadataExt, OpenOptionsExt}, +}; + +fn ready(owner: &mut Process, run: &str, deadline: Instant) { + loop { + assert!( + owner.poll().is_none(), + "foreground owner exited before ready" + ); + if let Some(end) = owner.out.iter().position(|byte| *byte == b'\n') { + let value: Value = serde_json::from_slice(&owner.out[..end]).unwrap(); + assert_eq!(value["kind"], "graph_foreground_ready"); + assert_eq!(value["run"], run); + return; + } + assert!(Instant::now() < deadline, "foreground readiness deadline"); + std::thread::sleep(Duration::from_millis(10)); + } +} + +fn legacy_pin(path: &Path, old: u64, before_boot: u64, control: bool) -> Vec { + let mut value: Value = state::read(path).unwrap(); + for key in if control { + ["parent", "endpoint"] + } else { + ["parent", "socket"] + } { + let pair = value[key].as_array_mut().unwrap(); + assert_ne!(pair[0].as_u64().unwrap(), old); + pair[0] = json!(old); + } + // This is a synthetic previous-physical-boot receipt. The actual owner + // exited before the test changes its private fixture bytes. + value["process"]["pid"] = json!(i32::MAX); + value["process"]["start_micros"] = json!(before_boot - 1); + state::write(path, &value).unwrap(); + fs::read(path).unwrap() +} + +fn refused_cli( + binary: &Path, + candidate: &Candidate, + args: &[&str], + expected_code: &str, + deadline: Instant, +) { + let mut process = Process::start(binary, candidate, args, None); + let status = process.wait(deadline); + assert_eq!( + status.code(), + Some(2), + "expected a CLI refusal, not a crash" + ); + assert!(process.out.is_empty(), "refused action emitted a result"); + let error: Value = serde_json::from_slice(&process.err).expect("structured CLI error"); + assert_eq!(error["code"], expected_code); + assert!(error["message"].as_str().is_some_and(|v| !v.is_empty())); +} + +fn publish_args<'a>(run: &'a str, expected: &'a str) -> [&'a str; 8] { + [ + "graph", + "recover-host-pins", + "--run-id", + run, + "--expect-selection", + expected, + "--accept-legacy-device-rebind", + "--json", + ] +} + +struct TemporarilyMoved { + original: PathBuf, + moved: PathBuf, +} +impl TemporarilyMoved { + fn new(original: PathBuf) -> Self { + let moved = original.with_extension("host-pin-test-held"); + assert!(!moved.exists()); + fs::rename(&original, &moved).unwrap(); + Self { original, moved } + } +} +impl Drop for TemporarilyMoved { + fn drop(&mut self) { + if self.moved.exists() { + assert!(!self.original.exists(), "foreign replacement at held path"); + fs::rename(&self.moved, &self.original).unwrap(); + } + } +} + +/// Preserve exact pre-mutation bytes outside managed state. A killed test can +/// be recovered from this private fixture artifact; normal unwinding restores +/// only the bytes this test itself wrote, never a foreign replacement. +struct ProviderReceiptRestore { + path: PathBuf, + original: Vec, + current: Vec, +} +impl ProviderReceiptRestore { + fn new(candidate: &Candidate, path: PathBuf) -> Self { + let original = fs::read(&path).unwrap(); + let backup = candidate + .checkout + .join("host-pin-test-provider-owner-original.json"); + let mut artifact = fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&backup) + .unwrap(); + artifact.write_all(&original).unwrap(); + artifact.sync_all().unwrap(); + fs::File::open(&candidate.checkout) + .unwrap() + .sync_all() + .unwrap(); + Self { + path, + current: original.clone(), + original, + } + } + fn replace(&mut self, bytes: Vec) { + assert_eq!(fs::read(&self.path).unwrap(), self.current); + fs::write(&self.path, &bytes).unwrap(); + self.current = bytes; + } + fn restore(&mut self) { + assert_eq!(fs::read(&self.path).unwrap(), self.current); + fs::write(&self.path, &self.original).unwrap(); + assert_eq!(fs::read(&self.path).unwrap(), self.original); + self.current = self.original.clone(); + } +} +impl Drop for ProviderReceiptRestore { + fn drop(&mut self) { + if self.current != self.original + && fs::read(&self.path).ok().as_deref() == Some(self.current.as_slice()) + { + let _ = fs::write(&self.path, &self.original); + } + } +} + +#[test] +#[ignore = "Owned capacity-two native VM, pinned static image and external 300s watchdog required"] +fn explicit_host_pin_witness_recovers_only_selected_previous_boot_run() { + let deadline = Instant::now() + Duration::from_secs(270); + let candidate = + Candidate::discover(Path::new(&std::env::var("HACK_LOCAL_TEST_ROOT").unwrap())).unwrap(); + let binary = PathBuf::from(std::env::var("HACK_LOCAL_TEST_BINARY").unwrap()); + let image = std::env::var("HACK_LOCAL_TEST_IMAGE").unwrap(); + let fixtures = [graph::tests::Fixture::new(), graph::tests::Fixture::new()]; + let runs = [ + graph::probes::token().unwrap(), + graph::probes::token().unwrap(), + ]; + let selection_root = graph::tests::Fixture::new(); + let mut plans = Vec::new(); + let mut selections = Vec::new(); + let mut dependencies = Vec::new(); + for (index, fixture) in fixtures.iter().enumerate() { + state::write(&fixture.0.join("compose.yaml"), &json!({ + "services":{"web":{"image":image,"read_only":true,"network_mode":"none", + "init":true,"user":"0:0","entrypoint":["/bin/sleep","300"],"command":[], + "volumes":["data:/data"],"healthcheck":{"test":["CMD","/bin/hack-graph-startup-app","complete"], + "interval":"200ms","timeout":"2s","retries":10,"start_period":"500ms"}}}, + "volumes":{"data":{}} + })).unwrap(); + let review = project::plan( + &candidate, + project::PlanOptions { + branch: None, + project: &fixture.0, + compose_file: Path::new("compose.yaml"), + profiles: &[], + }, + ) + .unwrap(); + let selection = selection_root.0.join(format!("dependencies-{index}.json")); + state::write(&selection, &json!({"version":1,"plan":review.plan_id, + "artifact":"/tmp/unused-control-only-artifact","artifact_sha256":"a".repeat(64),"dependencies":[]})).unwrap(); + let dependency = checked_cli( + "dependency-plan", + &binary, + &candidate, + &[ + "graph", + "dependency-plan", + "--dependencies", + selection.to_str().unwrap(), + "--json", + ], + deadline, + ); + plans.push(review); + selections.push(selection); + dependencies.push(dependency); + } + let start = |index: usize, generation: Option<&str>| { + let normalized = fixtures[index].0.join("compose.yaml"); + let mut args = vec![ + "graph", + if generation.is_some() { + "serve-restore" + } else { + "serve" + }, + "--project", + fixtures[index].0.to_str().unwrap(), + "--file", + "compose.yaml", + "--expect-plan", + &plans[index].plan_id, + "--run-id", + &runs[index], + "--ready", + "web=healthy", + "--timeout-seconds", + "90", + "--dependencies", + selections[index].to_str().unwrap(), + "--expect-dependencies", + dependencies[index]["dependency_plan_id"].as_str().unwrap(), + "--normalized-file", + normalized.to_str().unwrap(), + "--expect-original", + &plans[index].plan.compose_sha256, + "--expect-namespace", + &plans[index].plan.namespace, + "--json", + ]; + if let Some(generation) = generation { + args.extend(["--expect-generation", generation]); + } + Process::start(&binary, &candidate, &args, None) + }; + let mut selected_owner = start(0, None); + let mut selected_cleanup = Cleanup { + binary: &binary, + candidate: &candidate, + run: &runs[0], + done: false, + }; + ready(&mut selected_owner, &runs[0], deadline); + assert_eq!( + exec(&binary, &candidate, &runs[0], "web", "write-data", deadline)["exit_code"], + 0 + ); + let original = snapshot(&candidate, &runs[0], deadline).receipt; + assert_eq!(original.phase, "ready-observed"); + let expected_receipt = format!( + "{:x}", + Sha256::digest(serde_json::to_vec_pretty(&original).unwrap()) + ); + let old_boot = crate::provider::lifecycle::status(&candidate) + .unwrap() + .guest_boot_id + .unwrap(); + selected_owner.child.kill().unwrap(); + selected_owner.wait(deadline); + assert_eq!( + crate::provider::lifecycle::down(&candidate).unwrap().phase, + "stopped" + ); + let new_boot = crate::provider::lifecycle::up(&candidate) + .unwrap() + .guest_boot_id + .unwrap(); + assert_ne!(old_boot, new_boot); + + let mut sibling_owner = start(1, None); + let mut sibling_cleanup = Cleanup { + binary: &binary, + candidate: &candidate, + run: &runs[1], + done: false, + }; + ready(&mut sibling_owner, &runs[1], deadline); + assert_eq!( + exec(&binary, &candidate, &runs[1], "web", "write-data", deadline)["exit_code"], + 0 + ); + let sibling = serde_json::to_vec(&snapshot(&candidate, &runs[1], deadline).receipt).unwrap(); + + let root = graph::directory(&candidate, &runs[0]).unwrap(); + let graph_bytes = fs::read(root.join("state.json")).unwrap(); + let startup = original.relay_startup.as_ref().unwrap(); + let publisher_root = transport::root(&candidate, &runs[0]).unwrap(); + let publisher = publisher_root.join("owner.json"); + let control = startup.control_root.join("relay-control/owner.json"); + let current = fs::symlink_metadata(&publisher_root).unwrap().dev(); + let old = current.checked_add(1).unwrap(); + let before_boot = crate::provider::lifecycle::host_filesystem::host_boot_micros().unwrap(); + let publisher_bytes = legacy_pin(&publisher, old, before_boot, false); + let control_bytes = legacy_pin(&control, old, before_boot, true); + + // Ordinary recovery remains strict; the new witness is required to use + // the legacy device-only pins and is scoped to the selected run. + assert!(graph::foreground::transport::Pin::load(&candidate, &runs[0]).is_err()); + let inspected = checked_cli( + "inspect-host-pins", + &binary, + &candidate, + &[ + "graph", + "inspect-host-pin-recovery", + "--run-id", + &runs[0], + "--json", + ], + deadline, + ); + let selection = inspected["selection_sha256"].as_str().unwrap(); + assert_eq!(selection.len(), 64); + assert_eq!(inspected["old_device"], old); + assert_eq!(inspected["new_device"], current); + assert_eq!(fs::read(root.join("state.json")).unwrap(), graph_bytes); + assert_eq!(fs::read(&publisher).unwrap(), publisher_bytes); + assert_eq!(fs::read(&control).unwrap(), control_bytes); + + let witness_path = root.join("host-pin-recovery.json"); + let inspect_args = [ + "graph", + "inspect-host-pin-recovery", + "--run-id", + &runs[0], + "--json", + ]; + // A stale selection must not publish a witness or rewrite any source + // receipt. The three temporary path removals also preserve exact bytes. + refused_cli( + &binary, + &candidate, + &publish_args(&runs[0], &"0".repeat(64)), + "graph_host_pin_recovery", + deadline, + ); + assert!(!witness_path.exists()); + assert_eq!(fs::read(root.join("state.json")).unwrap(), graph_bytes); + assert_eq!(fs::read(&publisher).unwrap(), publisher_bytes); + assert_eq!(fs::read(&control).unwrap(), control_bytes); + { + let _missing = TemporarilyMoved::new(publisher_root.clone()); + refused_cli( + &binary, + &candidate, + &inspect_args, + "provider_state", + deadline, + ); + assert!(!witness_path.exists()); + } + assert_eq!(fs::read(&publisher).unwrap(), publisher_bytes); + { + let _missing = TemporarilyMoved::new(startup.control_root.join("relay-control")); + refused_cli( + &binary, + &candidate, + &inspect_args, + "provider_state", + deadline, + ); + assert!(!witness_path.exists()); + } + assert_eq!(fs::read(&control).unwrap(), control_bytes); + + // The provider receipt is valid JSON throughout. A changed raw byte + // selection and a changed guest boot must both refuse without writes. + let provider_path = candidate.state_root.join("run/smolvm/owner.json"); + let mut owner_restore = ProviderReceiptRestore::new(&candidate, provider_path.clone()); + let provider_bytes = owner_restore.original.clone(); + let mut whitespace = provider_bytes.clone(); + whitespace.push(b' '); + owner_restore.replace(whitespace.clone()); + refused_cli( + &binary, + &candidate, + &publish_args(&runs[0], selection), + "graph_host_pin_recovery", + deadline, + ); + assert_eq!(fs::read(&provider_path).unwrap(), whitespace); + assert!(!witness_path.exists()); + owner_restore.restore(); + let mut changed_boot: Value = serde_json::from_slice(&provider_bytes).unwrap(); + changed_boot["guest_boot_id"] = json!("f".repeat(32)); + assert_ne!(changed_boot["guest_boot_id"], new_boot); + owner_restore.replace(serde_json::to_vec_pretty(&changed_boot).unwrap()); + let boot_bytes = fs::read(&provider_path).unwrap(); + refused_cli( + &binary, + &candidate, + &inspect_args, + "graph_host_pin_recovery", + deadline, + ); + assert_eq!(fs::read(&provider_path).unwrap(), boot_bytes); + assert!(!witness_path.exists()); + owner_restore.restore(); + assert_eq!(fs::read(root.join("state.json")).unwrap(), graph_bytes); + assert_eq!(fs::read(&publisher).unwrap(), publisher_bytes); + assert_eq!(fs::read(&control).unwrap(), control_bytes); + + let refreshed = checked_cli( + "inspect-after-negative-controls", + &binary, + &candidate, + &inspect_args, + deadline, + ); + let selection = refreshed["selection_sha256"].as_str().unwrap(); + let recovered = checked_cli( + "publish-host-pin-witness", + &binary, + &candidate, + &publish_args(&runs[0], selection), + deadline, + ); + assert_eq!(recovered["witness_published"], true); + assert_eq!(fs::read(root.join("state.json")).unwrap(), graph_bytes); + assert_eq!(fs::read(&publisher).unwrap(), publisher_bytes); + assert_eq!(fs::read(&control).unwrap(), control_bytes); + assert_eq!( + serde_json::to_vec(&snapshot(&candidate, &runs[1], deadline).receipt).unwrap(), + sibling + ); + + let cleanup = checked_cli( + "legacy-dead-owner-cleanup", + &binary, + &candidate, + &[ + "graph", + "recover-cleanup", + "--run-id", + &runs[0], + "--expect-receipt", + &expected_receipt, + "--json", + ], + deadline, + ); + assert_eq!(cleanup["phase"], "stopped-data-retained"); + let retained = snapshot(&candidate, &runs[0], deadline).receipt; + assert_eq!( + retained.resources["volume:data"].name, + original.resources["volume:data"].name + ); + for _ in 0..2 { + let retired = checked_cli( + "legacy-publisher-retirement", + &binary, + &candidate, + &[ + "graph", + "retire-recovered-publisher", + "--run-id", + &runs[0], + "--expect-owner", + &retained.owner, + "--json", + ], + deadline, + ); + assert_eq!(retired["publisher_retired"], true); + } + let restoration = graph::foreground::restore_selection(&candidate, &runs[0]).unwrap(); + selected_owner = start(0, Some(restoration["generation"].as_str().unwrap())); + ready(&mut selected_owner, &runs[0], deadline); + assert_eq!( + exec(&binary, &candidate, &runs[0], "web", "read-data", deadline)["exit_code"], + 0, + "selected marker must survive cleanup, retirement and same-run restore" + ); + let restored = snapshot(&candidate, &runs[0], deadline).receipt; + assert_eq!( + restored.resources["volume:data"].name, + original.resources["volume:data"].name + ); + assert_ne!( + restored.resources["container:web"].id, + original.resources["container:web"].id + ); + assert!(sibling_owner.poll().is_none()); + assert_eq!( + serde_json::to_vec(&snapshot(&candidate, &runs[1], deadline).receipt).unwrap(), + sibling + ); + assert_eq!( + exec(&binary, &candidate, &runs[1], "web", "read-data", deadline)["exit_code"], + 0 + ); + let selected_removed = checked_cli( + "remove-selected-data", + &binary, + &candidate, + &[ + "graph", + "cleanup", + "--run-id", + &runs[0], + "--remove-data", + "--json", + ], + deadline, + ); + assert_eq!(selected_removed["phase"], "removed"); + assert!(selected_owner.wait(deadline).success()); + selected_cleanup.done = true; + let removed = checked_cli( + "remove-sibling-data", + &binary, + &candidate, + &[ + "graph", + "cleanup", + "--run-id", + &runs[1], + "--remove-data", + "--json", + ], + deadline, + ); + assert_eq!(removed["phase"], "removed"); + assert!(sibling_owner.wait(deadline).success()); + sibling_cleanup.done = true; +} diff --git a/packages/runtime-core/src/provider/graph/foreground/transport.rs b/packages/runtime-core/src/provider/graph/foreground/transport.rs index 8ab4a43f3..2064bfbc6 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport.rs @@ -1,5 +1,6 @@ use super::{Candidate, CandidateError, refused}; use crate::provider::{ + host_pin::DeviceRebind, identity::{self, ProcessIdentity}, state, }; @@ -239,6 +240,7 @@ fn selected_retirement_path( owner: &str, socket: bool, expected: (u64, u64), + device_rebind: Option, ) -> Result<(PathBuf, bool), CandidateError> { let original = root.join(if socket { "control.sock" } else { "owner.json" }); let retired = retired_path(root, owner, socket); @@ -247,8 +249,9 @@ fn selected_retirement_path( (None, Some(found)) => (retired, found, false), _ => return Err(retirement_refused()), }; - if id(¤t) != expected - || !private(¤t) + if !device_rebind.map_or(id(¤t) == expected, |v| { + v.matches(expected, id(¤t)) + }) || !private(¤t) || (socket && !current.file_type().is_socket()) || (!socket && (!current.is_file() || current.nlink() != 1 || current.len() > 8192)) { @@ -260,19 +263,26 @@ fn selected_retirement_path( fn verify_retirement( candidate: &Candidate, run: &str, - expected_owner: &str, - expected_receipt: &str, + expected: (&str, &str), root: &Path, lock: &state::Lock, intent: &Retirement, + device_rebind: Option, ) -> Result<(bool, bool), CandidateError> { + let root_id = id(&fs::symlink_metadata(root).map_err(|_| retirement_refused())?); + let lock_path_id = + id(&fs::symlink_metadata(root.join("operation.lock")).map_err(|_| retirement_refused())?); + let parent_matches = device_rebind.map_or(intent.parent == root_id, |v| { + v.matches(intent.parent, root_id) + }); if intent.version != 1 || intent.candidate != candidate.checkout || intent.run != run - || intent.owner_sha256 != expected_owner - || intent.receipt_sha256 != expected_receipt - || intent.parent != id(&fs::symlink_metadata(root).map_err(|_| retirement_refused())?) + || intent.owner_sha256 != expected.0 + || intent.receipt_sha256 != expected.1 + || !parent_matches || intent.lock != lock.identity().map_err(|_| retirement_refused())? + || lock_path_id != intent.lock || intent.owner.parent != intent.parent || intent.owner.socket != intent.socket || intent.owner.candidate != candidate.checkout @@ -282,9 +292,9 @@ fn verify_retirement( return Err(retirement_refused()); } let (socket_path, socket_original) = - selected_retirement_path(root, expected_owner, true, intent.socket)?; + selected_retirement_path(root, expected.0, true, intent.socket, device_rebind)?; let (record_path, record_original) = - selected_retirement_path(root, expected_owner, false, intent.record)?; + selected_retirement_path(root, expected.0, false, intent.record, None)?; // Socket retirement precedes record retirement. The inverse is foreign. if socket_original && !record_original { return Err(retirement_refused()); @@ -304,7 +314,7 @@ fn verify_retirement( .read_to_end(&mut bytes) .map_err(|_| retirement_refused())?; let record: Record = serde_json::from_slice(&bytes).map_err(|_| retirement_refused())?; - if record != intent.owner || format!("{:x}", Sha256::digest(&bytes)) != expected_owner { + if record != intent.owner || format!("{:x}", Sha256::digest(&bytes)) != expected.0 { return Err(retirement_refused()); } Ok((socket_original, record_original)) @@ -318,6 +328,15 @@ pub(in crate::provider::graph) fn retire_recovered_publisher( run: &str, expected_owner: &str, expected_receipt: &str, +) -> Result<(), CandidateError> { + retire_recovered_publisher_recovery(candidate, run, expected_owner, expected_receipt, None) +} +pub(in crate::provider::graph) fn retire_recovered_publisher_recovery( + candidate: &Candidate, + run: &str, + expected_owner: &str, + expected_receipt: &str, + device_rebind: Option, ) -> Result<(), CandidateError> { if !super::super::hex(expected_owner, 64) || !super::super::hex(expected_receipt, 64) { return Err(retirement_refused()); @@ -325,6 +344,24 @@ pub(in crate::provider::graph) fn retire_recovered_publisher( let root = root(candidate, run)?; state::check_private_directory(&root).map_err(|_| retirement_refused())?; let lock = state::Lock::acquire_existing(&root).map_err(|_| retirement_refused())?; + retire_recovered_publisher_locked( + candidate, + run, + expected_owner, + expected_receipt, + device_rebind, + &lock, + ) +} +pub(in crate::provider::graph) fn retire_recovered_publisher_locked( + candidate: &Candidate, + run: &str, + expected_owner: &str, + expected_receipt: &str, + device_rebind: Option, + lock: &state::Lock, +) -> Result<(), CandidateError> { + let root = root(candidate, run)?; let path = retirement_path(&root, expected_owner); let pending = path.with_extension("pending"); let name = pending @@ -335,7 +372,7 @@ pub(in crate::provider::graph) fn retire_recovered_publisher( let intent: Retirement = if metadata(&path)?.is_some() { state::read(&path).map_err(|_| retirement_refused())? } else { - let pin = Pin::read(candidate, run)?; + let pin = Pin::read_with_rebind(candidate, run, device_rebind)?; if format!("{:x}", Sha256::digest(&pin.bytes)) != expected_owner || identity::alive(pin.record.process.pid).unwrap_or(true) { @@ -360,11 +397,11 @@ pub(in crate::provider::graph) fn retire_recovered_publisher( let (socket_original, record_original) = verify_retirement( candidate, run, - expected_owner, - expected_receipt, + (expected_owner, expected_receipt), &root, - &lock, + lock, &intent, + device_rebind, )?; if socket_original { fs::rename( @@ -380,11 +417,11 @@ pub(in crate::provider::graph) fn retire_recovered_publisher( let _ = verify_retirement( candidate, run, - expected_owner, - expected_receipt, + (expected_owner, expected_receipt), &root, - &lock, + lock, &intent, + device_rebind, )?; fs::rename( root.join("owner.json"), @@ -398,11 +435,11 @@ pub(in crate::provider::graph) fn retire_recovered_publisher( let (socket_original, record_original) = verify_retirement( candidate, run, - expected_owner, - expected_receipt, + (expected_owner, expected_receipt), &root, - &lock, + lock, &intent, + device_rebind, )?; if socket_original || record_original { return Err(retirement_refused()); @@ -413,11 +450,27 @@ pub(in crate::provider::graph) fn retire_recovered_publisher( /** A later boot may resume frontend recovery only after retirement was fully * recorded. This read-only proof never begins or completes a partial move. */ +#[cfg(test)] pub(in crate::provider::graph) fn verify_recovered_publisher_retired( candidate: &Candidate, run: &str, expected_owner: &str, expected_receipt: &str, +) -> Result<(), CandidateError> { + verify_recovered_publisher_retired_recovery( + candidate, + run, + expected_owner, + expected_receipt, + None, + ) +} +pub(in crate::provider::graph) fn verify_recovered_publisher_retired_recovery( + candidate: &Candidate, + run: &str, + expected_owner: &str, + expected_receipt: &str, + device_rebind: Option, ) -> Result<(), CandidateError> { if !super::super::hex(expected_owner, 64) || !super::super::hex(expected_receipt, 64) { return Err(retirement_refused()); @@ -430,11 +483,11 @@ pub(in crate::provider::graph) fn verify_recovered_publisher_retired( let (socket_original, record_original) = verify_retirement( candidate, run, - expected_owner, - expected_receipt, + (expected_owner, expected_receipt), &root, &lock, &intent, + device_rebind, )?; if socket_original || record_original { return Err(retirement_refused()); @@ -485,11 +538,12 @@ fn peer(stream: &UnixStream) -> Result { identity::verify(&observed, &observed, &observed.executable, uid).map_err(|_| refused())?; Ok(observed) } -pub(super) struct Pin { +pub(in crate::provider::graph) struct Pin { root: PathBuf, record: Record, bytes: Vec, record_id: (u64, u64), + device_rebind: Option, } impl Pin { pub fn load(candidate: &Candidate, run: &str) -> Result { @@ -498,6 +552,13 @@ impl Pin { Ok(pin) } fn read(candidate: &Candidate, run: &str) -> Result { + Self::read_with_rebind(candidate, run, None) + } + fn read_with_rebind( + candidate: &Candidate, + run: &str, + device_rebind: Option, + ) -> Result { let root = root(candidate, run)?; state::check_private_directory(&root).map_err(|_| refused())?; let mut file = OpenOptions::new() @@ -530,6 +591,7 @@ impl Pin { record, bytes, record_id: id(&metadata), + device_rebind, }; pin.verify_files()?; Ok(pin) @@ -540,10 +602,10 @@ impl Pin { let socket = fs::symlink_metadata(self.root.join("control.sock")).map_err(|_| refused())?; let file = fs::symlink_metadata(self.root.join("owner.json")).map_err(|_| refused())?; if !parent.is_dir() - || id(&parent) != self.record.parent + || !self.matches_pin(self.record.parent, id(&parent)) || !socket.file_type().is_socket() || !private(&socket) - || id(&socket) != self.record.socket + || !self.matches_pin(self.record.socket, id(&socket)) || !file.is_file() || !private(&file) || file.nlink() != 1 @@ -570,6 +632,46 @@ impl Pin { } Ok(()) } + fn matches_pin(&self, recorded: (u64, u64), observed: (u64, u64)) -> bool { + self.device_rebind + .map_or(recorded == observed, |v| v.matches(recorded, observed)) + } + pub(in crate::provider::graph) fn legacy_summary( + candidate: &Candidate, + run: &str, + device_rebind: DeviceRebind, + host_boot_micros: u64, + ) -> Result { + let pin = Self::read_with_rebind(candidate, run, Some(device_rebind))?; + // SAFETY: geteuid has no arguments or side effects. + identity::verify( + &pin.record.process, + &pin.record.process, + &pin.record.process.executable, + unsafe { libc::geteuid() }, + )?; + device_rebind.definitely_dead_before_boot(&pin.record.process, host_boot_micros)?; + no_listener(&pin.root.join("control.sock"))?; + Ok(LegacyPublisher { + owner_sha256: format!("{:x}", Sha256::digest(&pin.bytes)), + parent: pin.record.parent, + socket: pin.record.socket, + record: pin.record_id, + process: pin.record.process, + }) + } + pub(in crate::provider::graph) fn legacy_recorded_device( + candidate: &Candidate, + run: &str, + ) -> Result { + let root = root(candidate, run)?; + state::check_private_directory(&root)?; + let record: Record = state::read_bounded(&root.join("owner.json"), 8192)?; + if record.version != 1 || record.candidate != candidate.checkout || record.run != run { + return Err(refused()); + } + Ok(record.parent.0) + } pub fn verify(&self) -> Result<(), CandidateError> { self.verify_files()?; let expected = &self.record.process; @@ -647,6 +749,16 @@ impl Pin { Ok(stream) } } + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(in crate::provider::graph) struct LegacyPublisher { + pub owner_sha256: String, + pub parent: (u64, u64), + pub socket: (u64, u64), + pub record: (u64, u64), + pub process: ProcessIdentity, +} pub(super) struct Publication { listener: UnixListener, pin: Pin, @@ -706,6 +818,7 @@ impl Publication { record, bytes, record_id, + device_rebind: None, }; pin.verify()?; Ok(Self { @@ -980,6 +1093,16 @@ impl Retired { run: &str, owner_sha256: &str, complete_sha256: &str, + ) -> Result<(), CandidateError> { + self.verify_recovery_with_rebind(candidate, run, owner_sha256, complete_sha256, None) + } + pub fn verify_recovery_with_rebind( + &self, + candidate: &Candidate, + run: &str, + owner_sha256: &str, + complete_sha256: &str, + device_rebind: Option, ) -> Result<(), CandidateError> { self.verify()?; if !super::super::hex(owner_sha256, 64) @@ -996,11 +1119,11 @@ impl Retired { let (socket_original, record_original) = verify_retirement( candidate, run, - owner_sha256, - complete_sha256, + (owner_sha256, complete_sha256), &self.root, &self._lock, &intent, + device_rebind, )?; if socket_original || record_original { return Err(retirement_refused()); @@ -1018,16 +1141,39 @@ impl DeadOwner { pub(in crate::provider::graph) fn acquire( candidate: &Candidate, run: &str, + ) -> Result { + Self::acquire_recovery(candidate, run, None, None) + } + pub(in crate::provider::graph) fn acquire_recovery( + candidate: &Candidate, + run: &str, + device_rebind: Option, + host_boot_micros: Option, ) -> Result { let lock = state::Lock::acquire_existing(&root(candidate, run)?)?; let value = Self { - pin: Pin::read(candidate, run)?, + pin: Pin::read_with_rebind(candidate, run, device_rebind)?, _lock: lock, }; + if let Some(rebind) = device_rebind { + rebind.definitely_dead_before_boot( + &value.pin.record.process, + host_boot_micros.ok_or_else(refused)?, + )?; + } value.verify()?; Ok(value) } pub(in crate::provider::graph) fn verify(&self) -> Result<(), CandidateError> { + let pathname = + fs::symlink_metadata(self.pin.root.join("operation.lock")).map_err(|_| refused())?; + if !pathname.is_file() + || pathname.nlink() != 1 + || !private(&pathname) + || id(&pathname) != self._lock.identity()? + { + return Err(refused()); + } self.pin.verify_files()?; if identity::alive(self.pin.record.process.pid)? { return Err(refused()); diff --git a/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs b/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs index 5b21b6a10..b81f11ee3 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs @@ -201,6 +201,78 @@ fn interrupted_socket_move_resumes_but_replaced_owner_refuses() { fs::remove_dir_all(root).unwrap(); } +#[cfg(target_os = "macos")] +#[test] +fn selected_legacy_device_socket_move_resumes_only_exact_retirement() { + let (_fixture, candidate, run, _original_owner, root) = abandoned_publisher(); + let receipt = "f".repeat(64); + let current = id(&fs::symlink_metadata(&root).unwrap()).0; + let rebind = DeviceRebind { + old: current.checked_add(1).unwrap(), + current, + }; + let mut record: Record = state::read(&root.join("owner.json")).unwrap(); + assert_eq!(record.parent.0, current); + assert_eq!(record.socket.0, current); + record.parent.0 = rebind.old; + record.socket.0 = rebind.old; + let bytes = serde_json::to_vec(&record).unwrap(); + fs::write(root.join("owner.json"), &bytes).unwrap(); + let owner = format!("{:x}", Sha256::digest(&bytes)); + assert!(Pin::read(&candidate, &run).is_err()); + let lock = state::Lock::acquire_existing(&root).unwrap(); + let pin = Pin::read_with_rebind(&candidate, &run, Some(rebind)).unwrap(); + let intent = Retirement { + version: 1, + candidate: candidate.checkout.clone(), + run: run.clone(), + receipt_sha256: receipt.clone(), + owner_sha256: owner.clone(), + parent: pin.record.parent, + lock: lock.identity().unwrap(), + socket: pin.record.socket, + record: pin.record_id, + owner: pin.record, + }; + state::write(&retirement_path(&root, &owner), &intent).unwrap(); + fs::rename(root.join("control.sock"), retired_path(&root, &owner, true)).unwrap(); + drop(lock); + assert!(retire_recovered_publisher(&candidate, &run, &owner, &receipt).is_err()); + retire_recovered_publisher_recovery(&candidate, &run, &owner, &receipt, Some(rebind)).unwrap(); + let retired = Retired::acquire(&candidate, &run).unwrap().unwrap(); + retired + .verify_recovery_with_rebind(&candidate, &run, &owner, &receipt, Some(rebind)) + .unwrap(); + assert!( + retired + .verify_recovery(&candidate, &run, &owner, &receipt) + .is_err() + ); + drop(retired); + fs::remove_dir_all(root).unwrap(); +} + +#[cfg(target_os = "macos")] +#[test] +fn dead_owner_guard_refuses_replaced_foreground_lock_path() { + let (_fixture, candidate, run, _owner, root) = abandoned_publisher(); + let dead = DeadOwner::acquire(&candidate, &run).unwrap(); + dead.verify().unwrap(); + let pathname = root.join("operation.lock"); + let saved = root.join("held-operation.lock"); + fs::rename(&pathname, &saved).unwrap(); + fs::write(&pathname, b"replacement").unwrap(); + fs::set_permissions(&pathname, fs::Permissions::from_mode(0o600)).unwrap(); + let replacement = id(&fs::symlink_metadata(&pathname).unwrap()); + assert!(dead.verify().is_err()); + assert_eq!(id(&fs::symlink_metadata(&pathname).unwrap()), replacement); + assert_eq!(fs::read(&pathname).unwrap(), b"replacement"); + drop(dead); + fs::remove_file(&pathname).unwrap(); + fs::rename(&saved, &pathname).unwrap(); + fs::remove_dir_all(root).unwrap(); +} + #[cfg(target_os = "macos")] #[test] fn replaced_socket_or_inherited_listener_refuses_retirement() { diff --git a/packages/runtime-core/src/provider/graph/host_pin_recovery.rs b/packages/runtime-core/src/provider/graph/host_pin_recovery.rs new file mode 100644 index 000000000..a41c94b31 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/host_pin_recovery.rs @@ -0,0 +1,704 @@ +//! Explicit, selected legacy host-pin migration for one retained graph run. +//! +//! The private witness precedes dead-owner cleanup. It never edits historical graph, +//! publisher, control, or dependency receipts and grants no ordinary replay authority. +use super::{Candidate, CandidateError, Engine, Kind, Receipt, directory, foreground, load, state}; +use crate::provider::{ + ProjectShareIntent, + host_pin::DeviceRebind, + lifecycle, + relay_owner::publication::{LegacyControl, PinnedEndpoint}, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + fs::{self, OpenOptions}, + io::{Read, Seek, SeekFrom}, + os::unix::fs::{MetadataExt, OpenOptionsExt}, + path::{Path, PathBuf}, +}; + +use super::{dependency_slots::LegacyReservation, foreground::transport::LegacyPublisher}; + +const FILE: &str = "host-pin-recovery.json"; +const LIMIT: u64 = 64 * 1024; +const GUEST_IDENTITY_ACK: &str = "host-pin-guest-identity-v1\n"; + +fn refused() -> CandidateError { + CandidateError::new( + "graph_host_pin_recovery", + "Legacy host pin selection is incomplete or changed; receipts and data were preserved.", + ) +} +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +/// Guest::boot_id is sourced from the host Owner receipt. Execute a read-only +/// guest check so a changed receipt cannot impersonate the running kernel boot. +/// execute_cleanup itself compares /proc boot ID and /storage owner under the +/// held Engine/Guest lease before this fixed acknowledgement is emitted. +fn verify_guest_identity(engine: &Engine<'_>) -> Result<(), CandidateError> { + let acknowledged = engine + .guest() + .execute_cleanup("printf 'host-pin-guest-identity-v1\\n'", &[]) + .map_err(|_| refused())?; + if acknowledged != GUEST_IDENTITY_ACK { + return Err(refused()); + } + Ok(()) +} +fn read_raw(path: &Path, limit: u64) -> Result, CandidateError> { + read_raw_with(path, limit, || {}) +} +fn read_raw_with( + path: &Path, + limit: u64, + after_read: impl FnOnce(), +) -> Result, CandidateError> { + let parent = path.parent().ok_or_else(refused)?; + state::check_private_directory(parent)?; + let mut file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path) + .map_err(|_| refused())?; + let metadata = file.metadata().map_err(|_| refused())?; + // SAFETY: geteuid has no arguments or side effects. + if !metadata.is_file() + || metadata.nlink() != 1 + || metadata.mode() & 0o7777 != 0o600 + || metadata.uid() != unsafe { libc::geteuid() } + || metadata.len() == 0 + || metadata.len() > limit + { + return Err(refused()); + } + let mut bytes = Vec::new(); + file.by_ref() + .take(limit + 1) + .read_to_end(&mut bytes) + .map_err(|_| refused())?; + after_read(); + file.seek(SeekFrom::Start(0)).map_err(|_| refused())?; + let mut confirmation = Vec::new(); + file.by_ref() + .take(limit + 1) + .read_to_end(&mut confirmation) + .map_err(|_| refused())?; + let retained = file.metadata().map_err(|_| refused())?; + let pathname = fs::symlink_metadata(path).map_err(|_| refused())?; + if bytes != confirmation + || bytes.len() as u64 != metadata.len() + || [retained.dev(), pathname.dev()] != [metadata.dev(); 2] + || [retained.ino(), pathname.ino()] != [metadata.ino(); 2] + || [retained.len(), pathname.len()] != [metadata.len(); 2] + || [retained.uid(), pathname.uid()] != [metadata.uid(); 2] + || [retained.mode(), pathname.mode()] != [metadata.mode(); 2] + || [retained.nlink(), pathname.nlink()] != [metadata.nlink(); 2] + || [retained.mtime(), pathname.mtime()] != [metadata.mtime(); 2] + || [retained.mtime_nsec(), pathname.mtime_nsec()] != [metadata.mtime_nsec(); 2] + || [retained.ctime(), pathname.ctime()] != [metadata.ctime(); 2] + || [retained.ctime_nsec(), pathname.ctime_nsec()] != [metadata.ctime_nsec(); 2] + { + return Err(refused()); + } + Ok(bytes) +} +fn absent(path: &Path) -> Result { + match fs::symlink_metadata(path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true), + _ => Err(refused()), + } +} + +fn exact_lock_path(root: &Path, held: &state::Lock) -> Result<(), CandidateError> { + let pathname = fs::symlink_metadata(root.join("operation.lock")).map_err(|_| refused())?; + if !pathname.is_file() + || pathname.nlink() != 1 + || (pathname.dev(), pathname.ino()) != held.identity()? + { + return Err(refused()); + } + Ok(()) +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Witness { + version: u8, + candidate: PathBuf, + run: String, + owner: String, + namespace: String, + plan: String, + graph_sha256: String, + provider_sha256: String, + host_boot_micros: u64, + previous_guest_boot: String, + current_guest_boot: String, + rebind: DeviceRebind, + publisher: LegacyPublisher, + control_root: PathBuf, + control: LegacyControl, + reservation: Option, + source_shared: Option, + retained_volumes: BTreeMap, + qualification: String, +} + +impl Witness { + pub(super) fn rebind(&self) -> DeviceRebind { + self.rebind + } + pub(super) fn host_boot_micros(&self) -> u64 { + self.host_boot_micros + } + pub(super) fn publisher_sha256(&self) -> &str { + &self.publisher.owner_sha256 + } + pub(super) fn reservation(&self) -> Option<&LegacyReservation> { + self.reservation.as_ref() + } + pub(super) fn control(&self) -> &LegacyControl { + &self.control + } + pub(super) fn control_root(&self) -> &Path { + &self.control_root + } + pub(super) fn graph_sha256(&self) -> &str { + &self.graph_sha256 + } + pub(super) fn matches_graph(&self, receipt: &Receipt) -> bool { + self.run == receipt.run + && self.owner == receipt.owner + && self.namespace == receipt.namespace + && self.plan == receipt.plan_id + && self.source_shared == receipt.source.as_ref().and_then(|s| s.shared.clone()) + } +} + +pub(super) struct Guard { + witness: Witness, + witness_path: PathBuf, + control_root: PathBuf, + control_lock: state::Lock, +} + +pub(super) struct CleanupProof<'a> { + pub original: &'a Receipt, + pub sha256: &'a str, + pub current_is_original: bool, + pub allow_absent_reservation: bool, + pub publisher_may_be_partial: bool, +} +impl Guard { + pub(super) fn witness(&self) -> &Witness { + &self.witness + } + pub(super) fn verify_lock(&self) -> Result<(), CandidateError> { + let retained: Witness = + serde_json::from_slice(&read_raw(&self.witness_path, LIMIT)?).map_err(|_| refused())?; + if retained != self.witness { + return Err(refused()); + } + let context = + super::host_relay::context(&self.witness.owner, &self.witness.previous_guest_boot)?; + let control = PinnedEndpoint::load_legacy_recovery( + &self.witness.control_root, + context, + self.witness.rebind, + self.witness.host_boot_micros, + )?; + if control.legacy_summary() != self.witness.control { + return Err(refused()); + } + exact_lock_path(&self.control_root, &self.control_lock) + } +} + +fn path(candidate: &Candidate, run: &str) -> Result { + Ok(directory(candidate, run)?.join(FILE)) +} +pub(super) fn load_witness( + candidate: &Candidate, + run: &str, +) -> Result, CandidateError> { + let file = path(candidate, run)?; + absent(&file.with_extension("pending"))?; + if absent(&file).is_ok() { + return Ok(None); + } + let bytes = read_raw(&file, LIMIT)?; + let witness: Witness = serde_json::from_slice(&bytes).map_err(|_| refused())?; + if witness.version != 1 || witness.candidate != candidate.checkout || witness.run != run { + return Err(refused()); + } + Ok(Some(witness)) +} + +/// Only the exact old publisher may select the historical overlay. A successor +/// with current-device pins uses ordinary strict recovery despite retained history. +pub(super) fn selected_for_old_publisher( + candidate: &Candidate, + run: &str, +) -> Result, CandidateError> { + let Some(witness) = load_witness(candidate, run)? else { + return Ok(None); + }; + let recorded = foreground::transport::Pin::legacy_recorded_device(candidate, run)?; + if recorded == witness.rebind.current { + return Ok(None); + } + if recorded != witness.rebind.old { + return Err(refused()); + } + Ok(Some(witness)) +} + +fn verify_volume_projections( + engine: &Engine<'_>, + receipt: &Receipt, + selected: &BTreeMap, +) -> Result<(), CandidateError> { + let keys = receipt + .resources + .iter() + .filter(|(_, value)| value.kind == Kind::Volume) + .map(|(key, _)| key.clone()) + .collect::>(); + if keys.len() != selected.len() || keys.iter().any(|key| !selected.contains_key(key)) { + return Err(refused()); + } + for key in keys { + let resource = &receipt.resources[&key]; + let actual = super::inspect_resource(engine, receipt, resource)?.ok_or_else(refused)?; + let expected = selected.get(&key).ok_or_else(refused)?; + if expected["expected_name"] != resource.name + || expected["observed_name"] != actual["Name"] + || expected["observed_created_at"] != actual["CreatedAt"] + || expected["observed_mountpoint"] != actual["Mountpoint"] + || expected["observed_driver"] != actual["Driver"] + || expected["observed_labels_sha256"] + != digest(&serde_json::to_vec(&actual["Labels"]).map_err(|_| refused())?) + { + return Err(refused()); + } + let labels = expected["labels"].as_object().ok_or_else(refused)?; + if labels + .iter() + .any(|(name, value)| actual["Labels"].get(name) != Some(value)) + { + return Err(refused()); + } + } + Ok(()) +} + +/// Caller already holds the Engine lease and exact foreground owner lock. +/// The returned control lock remains held across graph cleanup effects. +pub(super) fn acquire_for_cleanup( + candidate: &Candidate, + run: &str, + engine: &Engine<'_>, + proof: CleanupProof<'_>, + witness: Witness, +) -> Result { + lifecycle::host_filesystem::no_auxiliary_update(candidate)?; + if witness.graph_sha256 != proof.sha256 + || !witness.matches_graph(proof.original) + || witness.candidate != candidate.checkout + || witness.run != run + || witness.qualification + != "explicit-legacy-device-rebind-original-volume-continuity-unproven" + || lifecycle::host_filesystem::host_boot_micros()? != witness.host_boot_micros + { + return Err(refused()); + } + witness + .rebind + .definitely_dead_before_boot(&witness.publisher.process, witness.host_boot_micros)?; + if proof.current_is_original + && digest(&read_raw( + &directory(candidate, run)?.join("state.json"), + 2 * 1024 * 1024, + )?) != witness.graph_sha256 + { + return Err(refused()); + } + let owner = state::Owner::load(candidate)?; + lifecycle::verify_disks(candidate, &owner)?; + verify_guest_identity(engine)?; + let bytes = read_raw( + &candidate.state_root.join("run/smolvm/owner.json"), + 1024 * 1024, + )?; + if digest(&bytes) != witness.provider_sha256 + || owner.token != witness.owner + || owner.guest_boot_id.as_deref() != Some(&witness.current_guest_boot) + || owner.previous_guest_boot_id.as_deref() != Some(&witness.previous_guest_boot) + || engine.guest().incarnation() != witness.owner + || engine.guest().boot_id() != witness.current_guest_boot + || owner + .storage + .as_ref() + .is_none_or(|disk| disk.device != witness.rebind.current) + || owner + .overlay + .as_ref() + .is_none_or(|disk| disk.device != witness.rebind.current) + { + return Err(refused()); + } + if !proof.publisher_may_be_partial { + let publisher = foreground::transport::Pin::legacy_summary( + candidate, + run, + witness.rebind, + witness.host_boot_micros, + )?; + if publisher != witness.publisher { + return Err(refused()); + } + } + let control_root = witness.control_root.join("relay-control"); + let control_lock = state::Lock::acquire_existing(&control_root)?; + absent(&foreground::transport::root(candidate, run)?.join("owner.pending"))?; + absent(&control_root.join("owner.pending"))?; + let context = super::host_relay::context(&witness.owner, &witness.previous_guest_boot)?; + let control = PinnedEndpoint::load_legacy_recovery( + &witness.control_root, + context, + witness.rebind, + witness.host_boot_micros, + )? + .legacy_summary(); + if control != witness.control { + return Err(refused()); + } + if let Some(selected) = &witness.reservation { + super::dependency_slots::verify_legacy_remaining( + candidate, + run, + witness.rebind, + selected, + proof.allow_absent_reservation, + )?; + } else if super::dependency_slots::inspect_legacy( + candidate, + run, + witness.rebind, + witness.host_boot_micros, + )? + .is_some() + { + return Err(refused()); + } + verify_volume_projections(engine, proof.original, &witness.retained_volumes)?; + let guard = Guard { + witness_path: path(candidate, run)?, + witness, + control_root, + control_lock, + }; + guard.verify_lock()?; + Ok(guard) +} + +struct Selected { + witness: Witness, + foreground_root: PathBuf, + foreground_lock: state::Lock, + control_lock_root: PathBuf, + control_lock: state::Lock, +} + +impl Selected { + fn verify( + &self, + candidate: &Candidate, + run: &str, + engine: &Engine<'_>, + ) -> Result<(), CandidateError> { + for (root, lock) in [ + (&self.foreground_root, &self.foreground_lock), + (&self.control_lock_root, &self.control_lock), + ] { + exact_lock_path(root, lock)?; + } + if self.foreground_root != foreground::transport::root(candidate, run)? + || self.control_lock_root != self.witness.control_root.join("relay-control") + || self.witness != select_content(candidate, run, engine)? + { + return Err(refused()); + } + Ok(()) + } +} + +fn select( + candidate: &Candidate, + run: &str, + engine: &Engine<'_>, +) -> Result { + let (receipt, _) = load(candidate, engine, run)?; + let startup = receipt.relay_startup.as_ref().ok_or_else(refused)?; + let foreground_root = foreground::transport::root(candidate, run)?; + let foreground_lock = state::Lock::acquire_existing(&foreground_root)?; + let control_lock_root = startup.control_root.join("relay-control"); + let control_lock = state::Lock::acquire_existing(&control_lock_root)?; + let selected = Selected { + witness: select_content(candidate, run, engine)?, + foreground_root, + foreground_lock, + control_lock_root, + control_lock, + }; + selected.verify(candidate, run, engine)?; + Ok(selected) +} + +fn select_content( + candidate: &Candidate, + run: &str, + engine: &Engine<'_>, +) -> Result { + lifecycle::host_filesystem::no_auxiliary_update(candidate)?; + let owner = state::Owner::load(candidate)?; + lifecycle::verify_disks(candidate, &owner)?; + verify_guest_identity(engine)?; + let host_boot_micros = lifecycle::host_filesystem::host_boot_micros()?; + let current_guest_boot = owner.guest_boot_id.clone().ok_or_else(refused)?; + let previous_guest_boot = owner.previous_guest_boot_id.clone().ok_or_else(refused)?; + if current_guest_boot == previous_guest_boot + || current_guest_boot != engine.guest().boot_id() + || owner.token != engine.guest().incarnation() + { + return Err(refused()); + } + let (receipt, root) = load(candidate, engine, run)?; + for name in [ + "state.pending", + "dead-owner-cleanup.pending", + "relay-cleanup-bridges.pending", + ] { + absent(&root.join(name))?; + } + if receipt.phase != "ready-observed" || receipt.relay_cleanup.is_some() { + return Err(refused()); + } + let startup = receipt.relay_startup.as_ref().ok_or_else(refused)?; + let graph_sha256 = digest(&read_raw(&root.join("state.json"), 2 * 1024 * 1024)?); + let provider_sha256 = digest(&read_raw( + &candidate.state_root.join("run/smolvm/owner.json"), + 1024 * 1024, + )?); + let publisher_root = foreground::transport::root(candidate, run)?; + absent(&publisher_root.join("owner.pending"))?; + absent(&startup.control_root.join("relay-control/owner.pending"))?; + let current_device = fs::symlink_metadata(&publisher_root) + .map_err(|_| refused())? + .dev(); + let rebind = DeviceRebind { + old: foreground::transport::Pin::legacy_recorded_device(candidate, run)?, + current: current_device, + }; + if rebind.old == rebind.current { + return Err(refused()); + } + if owner + .storage + .as_ref() + .is_none_or(|disk| disk.device != rebind.current) + || owner + .overlay + .as_ref() + .is_none_or(|disk| disk.device != rebind.current) + { + return Err(refused()); + } + let publisher = + foreground::transport::Pin::legacy_summary(candidate, run, rebind, host_boot_micros)?; + let context = super::host_relay::context(&receipt.owner, &previous_guest_boot)?; + let control = PinnedEndpoint::load_legacy_recovery( + &startup.control_root, + context, + rebind, + host_boot_micros, + )? + .legacy_summary(); + let reservation = + super::dependency_slots::inspect_legacy(candidate, run, rebind, host_boot_micros)?; + if receipt + .source + .as_ref() + .and_then(|source| source.shared.as_ref()) + .is_some_and(|shared| shared.device != rebind.old) + || owner.project_share.as_ref() + != receipt + .source + .as_ref() + .and_then(|s| s.shared.as_ref()) + .map(|shared| { + // Historical shared source is pinned to the old host device. Only the + // selected device may differ from the currently approved project share. + let mut expected = shared.clone(); + expected.device = rebind.current; + expected + }) + .as_ref() + { + return Err(refused()); + } + if let Some(share) = &owner.project_share { + share.validate()?; + } + let mut retained_volumes = BTreeMap::new(); + for (key, resource) in receipt + .resources + .iter() + .filter(|(_, value)| value.kind == Kind::Volume) + { + let actual = super::inspect_resource(engine, &receipt, resource)?.ok_or_else(refused)?; + let labels = super::expected_labels(&receipt, resource) + .as_object() + .ok_or_else(refused)? + .keys() + .map(|label| (label.clone(), actual["Labels"][label].clone())) + .collect::>(); + retained_volumes.insert( + key.clone(), + json!({"expected_name":resource.name,"observed_name":actual["Name"], + "observed_created_at":actual["CreatedAt"], + "observed_mountpoint":actual["Mountpoint"], + "observed_driver":actual["Driver"], + "observed_labels_sha256":digest(&serde_json::to_vec(&actual["Labels"]).map_err(|_| refused())?), + "labels":labels}), + ); + } + Ok(Witness { + version: 1, + candidate: candidate.checkout.clone(), + run: run.into(), + owner: receipt.owner, + namespace: receipt.namespace, + plan: receipt.plan_id, + graph_sha256, + provider_sha256, + host_boot_micros, + previous_guest_boot, + current_guest_boot, + rebind, + publisher, + control_root: startup.control_root.clone(), + control, + reservation, + source_shared: receipt.source.and_then(|s| s.shared), + retained_volumes, + qualification: "explicit-legacy-device-rebind-original-volume-continuity-unproven".into(), + }) +} + +fn selected_sha256(witness: &Witness) -> Result { + Ok(digest(&serde_json::to_vec(witness).map_err(|_| refused())?)) +} + +/// Read-only exact selection. Existing completed recovery history is retained. +pub fn inspect(candidate: &Candidate, run: &str) -> Result { + let engine = Engine::connect_cleanup_wait(candidate)?; + let selected = select(candidate, run, &engine)?; + Ok( + json!({"run":run,"selection_sha256":selected_sha256(&selected.witness)?,"old_device":selected.witness.rebind.old,"new_device":selected.witness.rebind.current,"qualification":selected.witness.qualification}), + ) +} + +/// Explicitly publish one private immutable witness before cleanup can consume an overlay. +pub fn recover(candidate: &Candidate, run: &str, expected: &str) -> Result { + if expected.len() != 64 || !expected.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(refused()); + } + let engine = Engine::connect_cleanup_wait(candidate)?; + let selected = select(candidate, run, &engine)?; + if selected_sha256(&selected.witness)? != expected { + return Err(refused()); + } + if let Some(existing) = load_witness(candidate, run)? { + if existing == selected.witness { + return Ok( + json!({"run":run,"selection_sha256":expected,"witness_published":true,"already_published":true,"qualification":selected.witness.qualification}), + ); + } + return Err(refused()); + } + selected.verify(candidate, run, &engine)?; + state::write(&path(candidate, run)?, &selected.witness)?; + Ok( + json!({"run":run,"selection_sha256":expected,"witness_published":true,"qualification":selected.witness.qualification}), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{io::Write, os::unix::fs::PermissionsExt}; + + #[test] + fn raw_selection_refuses_same_inode_in_place_substitution() { + let fixture = super::super::tests::Fixture::new(); + let path = fixture.0.join("selected.json"); + fs::write(&path, b"first").unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + let before = fs::symlink_metadata(&path).unwrap(); + let result = read_raw_with(&path, 64, || { + let mut file = OpenOptions::new().write(true).open(&path).unwrap(); + file.write_all(b"other").unwrap(); + file.sync_all().unwrap(); + }); + assert!(result.is_err()); + assert_eq!(fs::symlink_metadata(&path).unwrap().ino(), before.ino()); + } + + #[test] + fn device_overlay_requires_only_one_number_change_and_exact_inode() { + let translated = DeviceRebind { + old: 10, + current: 20, + }; + assert!(translated.matches((10, 77), (20, 77))); + for (old, current) in [ + ((20, 77), (20, 77)), + ((10, 77), (10, 77)), + ((10, 77), (20, 78)), + ((10, 0), (20, 0)), + ] { + assert!(!translated.matches(old, current)); + } + } + + #[test] + fn held_lock_path_replacement_refuses_and_preserves_foreign_replacement() { + let fixture = super::super::tests::Fixture::new(); + let lock = state::Lock::acquire(&fixture.0).unwrap(); + exact_lock_path(&fixture.0, &lock).unwrap(); + let pathname = fixture.0.join("operation.lock"); + let saved = fixture.0.join("held-operation.lock"); + fs::rename(&pathname, &saved).unwrap(); + fs::write(&pathname, b"foreign replacement").unwrap(); + fs::set_permissions(&pathname, fs::Permissions::from_mode(0o600)).unwrap(); + let replacement = fs::symlink_metadata(&pathname).unwrap(); + assert!(exact_lock_path(&fixture.0, &lock).is_err()); + let after = fs::symlink_metadata(&pathname).unwrap(); + assert_eq!( + (after.dev(), after.ino()), + (replacement.dev(), replacement.ino()) + ); + assert_eq!(fs::read(&pathname).unwrap(), b"foreign replacement"); + drop(lock); + // The fixture is private and isolated; restore the original pathname + // before its directory is torn down. + fs::remove_file(&pathname).unwrap(); + fs::rename(&saved, &pathname).unwrap(); + } +} diff --git a/packages/runtime-core/src/provider/graph/host_relay.rs b/packages/runtime-core/src/provider/graph/host_relay.rs index 94a5145d8..49946e7f5 100644 --- a/packages/runtime-core/src/provider/graph/host_relay.rs +++ b/packages/runtime-core/src/provider/graph/host_relay.rs @@ -489,6 +489,25 @@ pub(super) fn inspect_cleanup( remove_data: bool, environment: &super::super::environment_recovery::GraphInventory, bridge_selection: &bridges::cleanup::Selection, +) -> Result<[u8; 32], CandidateError> { + inspect_cleanup_recovery( + candidate, + engine, + expected, + remove_data, + environment, + bridge_selection, + None, + ) +} +pub(super) fn inspect_cleanup_recovery( + candidate: &Candidate, + engine: &Engine<'_>, + expected: &Receipt, + remove_data: bool, + environment: &super::super::environment_recovery::GraphInventory, + bridge_selection: &bridges::cleanup::Selection, + host_pin: Option<&super::host_pin_recovery::Witness>, ) -> Result<[u8; 32], CandidateError> { let (receipt, root) = archive::load_confirmation(candidate, engine, &expected.run)?; match fs::symlink_metadata(root.join("state.pending")) { @@ -533,10 +552,10 @@ pub(super) fn inspect_cleanup( require_retained_volume(resource, value.is_some(), remove_data)?; observations.insert(key, value); } - if &bridges::cleanup::read(engine, &receipt, &root)? != bridge_selection { + if &bridges::cleanup::read_recovery(engine, &receipt, &root, host_pin)? != bridge_selection { return Err(refused()); } - bridges::cleanup::verify(candidate, engine, &receipt, bridge_selection)?; + bridges::cleanup::verify_recovery(candidate, engine, &receipt, bridge_selection, host_pin)?; environment::verify_cleanup(candidate, engine, &receipt, &root, environment)?; probes::verify_cleanup(engine, &receipt)?; startup::verify_cleanup(engine, &receipt)?; diff --git a/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs b/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs index 33f1aa01e..d8b89dc68 100644 --- a/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs +++ b/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs @@ -380,7 +380,7 @@ fn finish_retirement( &intent.foreground_sha256, &digest(receipt)?, )?; - dependency_slots::recover_cleaned(candidate, receipt)?; + dependency_slots::recover_cleaned(candidate, receipt, None)?; engine.guest().verify() } diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index 0236bce3c..d43122f1e 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -18,9 +18,13 @@ mod dependency_hosts; #[cfg(target_os = "macos")] mod dependency_slots; #[cfg(target_os = "macos")] +mod host_pin_recovery; +#[cfg(target_os = "macos")] pub use dependency_slots::{ inspect as dependency_reservations, recover_orphan as recover_dependency_reservation, }; +#[cfg(target_os = "macos")] +pub use host_pin_recovery::{inspect as inspect_host_pin_recovery, recover as recover_host_pins}; mod initializer_cache; mod volume_subpaths; pub use dependency_hosts::dependency_address; diff --git a/packages/runtime-core/src/provider/host_pin.rs b/packages/runtime-core/src/provider/host_pin.rs new file mode 100644 index 000000000..f583198d4 --- /dev/null +++ b/packages/runtime-core/src/provider/host_pin.rs @@ -0,0 +1,40 @@ +//! Exact device-number translation for explicitly selected pre-reboot host pins. +//! This is never used by ordinary ownership, source replay, or guest observations. +use super::identity::{self, ProcessIdentity}; +use crate::CandidateError; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct DeviceRebind { + pub old: u64, + pub current: u64, +} + +impl DeviceRebind { + pub fn matches(self, recorded: (u64, u64), observed: (u64, u64)) -> bool { + self.old != self.current + && recorded.0 == self.old + && observed.0 == self.current + && recorded.1 != 0 + && recorded.1 == observed.1 + } + + pub fn definitely_dead_before_boot( + self, + process: &ProcessIdentity, + host_boot_micros: u64, + ) -> Result<(), CandidateError> { + if host_boot_micros == 0 + || process.start_micros == 0 + || process.start_micros >= host_boot_micros + || identity::alive(process.pid)? + { + return Err(CandidateError::new( + "host_pin_recovery", + "Legacy host pin owner is not proved dead before this host boot.", + )); + } + Ok(()) + } +} diff --git a/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs b/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs index 6c9e10634..2b7ad0ddb 100644 --- a/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs +++ b/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs @@ -35,7 +35,7 @@ fn refused(detail: &str) -> CandidateError { } #[cfg(target_os = "macos")] -fn host_boot_micros() -> Result { +pub(crate) fn host_boot_micros() -> Result { let mut time = std::mem::MaybeUninit::::zeroed(); let mut length = std::mem::size_of::(); // SAFETY: read-only sysctl writes an exactly sized timeval; no input or retained pointers. @@ -74,7 +74,7 @@ fn host_boot_micros() -> Result { )) } -fn no_auxiliary_update(candidate: &Candidate) -> Result<(), CandidateError> { +pub(crate) fn no_auxiliary_update(candidate: &Candidate) -> Result<(), CandidateError> { crate::provider::network_update::require_complete(candidate)?; for name in [ "owner.pending", diff --git a/packages/runtime-core/src/provider/mod.rs b/packages/runtime-core/src/provider/mod.rs index e851f1890..d1203b751 100644 --- a/packages/runtime-core/src/provider/mod.rs +++ b/packages/runtime-core/src/provider/mod.rs @@ -19,6 +19,8 @@ pub mod environment; mod environment_probe_test; pub mod environment_recovery; pub mod graph; +#[cfg(target_os = "macos")] +mod host_pin; pub use engine::{EngineInfo, info as engine_info}; #[cfg(all(test, target_os = "macos", target_arch = "aarch64"))] mod gateway_probe_test; diff --git a/packages/runtime-core/src/provider/relay_owner/publication.rs b/packages/runtime-core/src/provider/relay_owner/publication.rs index 2210b12ec..178e0a97d 100644 --- a/packages/runtime-core/src/provider/relay_owner/publication.rs +++ b/packages/runtime-core/src/provider/relay_owner/publication.rs @@ -7,6 +7,7 @@ use super::{ use crate::{ CandidateError, provider::{ + host_pin::DeviceRebind, identity::{self, ProcessIdentity}, state, }, @@ -102,6 +103,16 @@ pub struct PinnedEndpoint { receipt: Receipt, receipt_id: FileId, bytes: Vec, + device_rebind: Option, +} +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct LegacyControl { + pub receipt_sha256: String, + pub receipt_id: FileId, + pub parent: FileId, + pub endpoint: FileId, + pub process: ProcessIdentity, } impl PinnedEndpoint { pub(crate) fn runtime_root(&self) -> &Path { @@ -134,6 +145,13 @@ impl PinnedEndpoint { Self::read(Paths::new(root)?, context) } fn read(paths: Paths, context: Context) -> Result { + Self::read_with_rebind(paths, context, None) + } + fn read_with_rebind( + paths: Paths, + context: Context, + device_rebind: Option, + ) -> Result { let parent = paths.parent()?; let mut file = OpenOptions::new() .read(true) @@ -160,7 +178,9 @@ impl PinnedEndpoint { || context.boot == [0; 16] || receipt.owner == [0; 16] || receipt.socket != paths.socket - || receipt.parent != parent + || !device_rebind.map_or(receipt.parent == parent, |v| { + v.matches(receipt.parent, parent) + }) || receipt.endpoint.1 == 0 || !receipt.process.executable.is_absolute() { @@ -179,8 +199,30 @@ impl PinnedEndpoint { receipt, receipt_id: id(&m), bytes, + device_rebind, }) } + pub(crate) fn load_legacy_recovery( + root: &Path, + context: Context, + device_rebind: DeviceRebind, + host_boot_micros: u64, + ) -> Result { + let pin = Self::read_with_rebind(Paths::new(root)?, context, Some(device_rebind))?; + pin.verify_dead()?; + device_rebind.definitely_dead_before_boot(&pin.receipt.process, host_boot_micros)?; + dead::no_listener(&pin.paths.socket)?; + Ok(pin) + } + pub(crate) fn legacy_summary(&self) -> LegacyControl { + LegacyControl { + receipt_sha256: format!("{:x}", Sha256::digest(&self.bytes)), + receipt_id: self.receipt_id, + parent: self.receipt.parent, + endpoint: self.receipt.endpoint, + process: self.receipt.process.clone(), + } + } /// Read-only predecessor proof: exact private receipt/socket, absent owner. #[cfg(target_os = "macos")] pub(crate) fn verify_dead(&self) -> Result<(), CandidateError> { @@ -198,12 +240,13 @@ impl PinnedEndpoint { Sha256::digest(&self.bytes).into() } fn verify_receipt(&self) -> Result<(), CandidateError> { - let current = Self::read( + let current = Self::read_with_rebind( self.paths.clone(), Context { runtime: self.receipt.runtime, boot: self.receipt.boot, }, + self.device_rebind, )?; if current.receipt_id != self.receipt_id || current.bytes != self.bytes { return Err(refused()); @@ -211,14 +254,24 @@ impl PinnedEndpoint { Ok(()) } fn verify_socket(&self) -> Result<(), CandidateError> { - if self.paths.parent()? != self.receipt.parent { + let parent = self.paths.parent()?; + if !self + .device_rebind + .map_or(parent == self.receipt.parent, |v| { + v.matches(self.receipt.parent, parent) + }) + { return Err(refused()); } let m = fs::symlink_metadata(&self.paths.socket).map_err(|_| refused())?; if !m.file_type().is_socket() || !private(&m) || m.nlink() != 1 - || id(&m) != self.receipt.endpoint + || !self + .device_rebind + .map_or(id(&m) == self.receipt.endpoint, |v| { + v.matches(self.receipt.endpoint, id(&m)) + }) { return Err(refused()); } @@ -353,6 +406,7 @@ impl ControlListener { receipt, receipt_id, bytes, + device_rebind: None, }; endpoint.verify_receipt()?; endpoint.verify_socket()?; diff --git a/packages/runtime-core/src/provider/relay_owner/publication/dead.rs b/packages/runtime-core/src/provider/relay_owner/publication/dead.rs index edec289f5..4c5390feb 100644 --- a/packages/runtime-core/src/provider/relay_owner/publication/dead.rs +++ b/packages/runtime-core/src/provider/relay_owner/publication/dead.rs @@ -41,6 +41,7 @@ impl Selection { receipt, receipt_id: self.record_id, bytes: self.bytes.clone(), + device_rebind: None, }) } } From 2404e436492213c104706d7589ff86735f6c2505 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 02:44:15 -0400 Subject: [PATCH 05/25] test(runtime): model absent publication recovery safety --- scripts/lib/tla-runtime-models.ts | 55 +++++++ tests/models/tla/README.md | 41 ++++++ .../absent-publication-recovery/Absent.tla | 136 ++++++++++++++++++ .../absent-publication-recovery/negative.cfg | 12 ++ .../absent-publication-recovery/positive.cfg | 12 ++ .../stale-selection.cfg | 12 ++ .../unconfirmed-retirement.cfg | 12 ++ .../unwitnessed-cleanup.cfg | 12 ++ 8 files changed, 292 insertions(+) create mode 100644 tests/models/tla/absent-publication-recovery/Absent.tla create mode 100644 tests/models/tla/absent-publication-recovery/negative.cfg create mode 100644 tests/models/tla/absent-publication-recovery/positive.cfg create mode 100644 tests/models/tla/absent-publication-recovery/stale-selection.cfg create mode 100644 tests/models/tla/absent-publication-recovery/unconfirmed-retirement.cfg create mode 100644 tests/models/tla/absent-publication-recovery/unwitnessed-cleanup.cfg diff --git a/scripts/lib/tla-runtime-models.ts b/scripts/lib/tla-runtime-models.ts index c0222485e..f2fd5799c 100644 --- a/scripts/lib/tla-runtime-models.ts +++ b/scripts/lib/tla-runtime-models.ts @@ -26,6 +26,61 @@ type ModelContract = { // Bounds and witnesses are reviewed contracts, not learned from each run. const contracts: readonly ModelContract[] = [ + { + name: "absent-publication-recovery", + module: "Absent", + states: 247, + invariant: "NoPrematurePublication", + action: "Publish", + fields: [ + "intent = TRUE", + "complete = FALSE", + "retired = FALSE", + "published = TRUE", + "unsafePublication = TRUE", + ], + additionalControls: [ + { + name: "unwitnessed-cleanup", + negative: true, + invariant: "NoUnwitnessedCleanup", + action: "CleanupOne", + fields: [ + "intent = FALSE", + "progress = 1", + 'engine = "recovery"', + 'foreground = "recovery"', + "unsafeCleanup = TRUE", + ], + }, + { + name: "stale-selection", + negative: true, + invariant: "NoUnwitnessedCleanup", + action: "CleanupOne", + fields: [ + "intent = TRUE", + "selected = 1", + "version = 2", + "progress = 1", + "unsafeCleanup = TRUE", + ], + }, + { + name: "unconfirmed-retirement", + negative: true, + invariant: "NoUnconfirmedRetirement", + action: "RetireAbsentPublisher", + fields: [ + "intent = FALSE", + "complete = FALSE", + "progress = 0", + "retired = TRUE", + "unsafeRetirement = TRUE", + ], + }, + ], + }, { name: "shared-https-lifetime", module: "SharedHttps", diff --git a/tests/models/tla/README.md b/tests/models/tla/README.md index cd08ad461..358a5d65a 100644 --- a/tests/models/tla/README.md +++ b/tests/models/tla/README.md @@ -16,6 +16,47 @@ pinned artifact. Each check has a 120-second timeout, 512 MiB Java heap, two wor and bounded output; temporary TLC metadata is removed after success or failure. No credentials or running VM are needed. +## Missing post-reboot publications + +`absent-publication-recovery/Absent.tla` checks one explicitly selected cleanup +racing one foreground publisher. The positive configuration explores 247 distinct +states, including one recovery-process crash, two non-volume cleanup effects and +one input-version change. Both participants acquire the foreground lock before the +Engine lease. Ordinary publication occurs under the foreground lock before Engine +admission, matching the implementation. The durable absence intent survives a +crash; a restart cannot publish until cleanup and separate absence retirement are +confirmed. A changed selection +cannot resume cleanup. Completion and retirement are separate steps, so a crash +between them remains visible. + +Four guard-removal controls require TLC exit 12 and a named same-state witness: + +| Control | Required failure | +| --- | --- | +| `negative` | `NoPrematurePublication` in `Publish`, with a durable intent, incomplete cleanup, no retirement and a newly published owner | +| `unwitnessed-cleanup` | `NoUnwitnessedCleanup` in `CleanupOne`, with both locks held but no intent | +| `stale-selection` | `NoUnwitnessedCleanup` in `CleanupOne`, with a durable intent but selected version 1 and current version 2 | +| `unconfirmed-retirement` | `NoUnconfirmedRetirement` in `RetireAbsentPublisher`, before cleanup or intent | + +| Model action | Implementation boundary under `packages/runtime-core/src/provider/` | +| --- | --- | +| `AcquireRecoveryForeground` / `AcquireRecoveryEngine` | `graph/absent_publication_cleanup.rs`: deterministic lock-only reservation before the Engine lease; matches foreground startup's lock order | +| `WriteIntent` | Private durable run-scoped absence intent before any cleanup effects | +| `CleanupOne` / `CommitCleanup` | Exact selected, data-retaining `cleanup_owned(..., false)` and stopped receipt confirmation; tagged absent bridge authority remains distinct from pinned predecessors | +| `RetireAbsentPublisher` | Separate durable absent-publication retirement proof tied to completed cleanup | +| `AcquirePublisherForeground` / `AcquirePublisherEngine` / `Publish` | `graph/foreground/transport.rs` admission barrier plus `graph/foreground.rs` Engine acquisition; explicit retired binding needs completed proof | +| `CrashRecovery` / `ChangeInputs` | Retained intent and exact retry; reinspection refuses changed host/guest boot, input bytes or resources | + +The model assumes validated legacy inputs, absent roots and resource observations +are summarized by one version. It represents cooperating writers under two kernel +locks and durable writes as atomic commits. It does not prove filesystem fsync, +FD/path identity, raw hashes, PID reuse, private permissions, actual guest or +physical host reboot, bridge cleanup, volume preservation, source-device migration, +later restore generations, browser readiness or performance. Concrete refusal, +interruption and data-marker tests remain required. The recorded old Owner is +legacy corroboration, never a reconstructed foreground Pin or proof of original +physical-volume continuity. No fairness or eventual recovery claim is made. + ## Shared HTTPS lifetime `shared-https-lifetime/SharedHttps.tla` checks the last-lease release racing diff --git a/tests/models/tla/absent-publication-recovery/Absent.tla b/tests/models/tla/absent-publication-recovery/Absent.tla new file mode 100644 index 000000000..a736a3182 --- /dev/null +++ b/tests/models/tla/absent-publication-recovery/Absent.tla @@ -0,0 +1,136 @@ +----------------------------- MODULE Absent ----------------------------- +EXTENDS Naturals, TLC +CONSTANTS EnforceIntent, EnforceBarrier, EnforceSelection, EnforceCompletion +VARIABLES foreground, engine, recovery, publisher, intent, progress, + complete, retired, published, version, selected, crashed, + unsafeCleanup, unsafePublication, unsafeRetirement +durable == <> +observations == <> +effects == <> +vars == <> +LocksHeld == foreground = "recovery" /\ engine = "recovery" +SelectionMatches == selected = version +\* Exact legacy inputs and resource/guest observations are one abstract version. +\* No old Pin is synthesized. The selection starts without any live publication. +Init == /\ foreground = "none" /\ engine = "none" + /\ recovery = "idle" /\ publisher = "idle" + /\ intent = FALSE /\ progress = 0 /\ complete = FALSE + /\ retired = FALSE /\ published = FALSE + /\ version = 1 /\ selected = 0 /\ crashed = FALSE + /\ unsafeCleanup = FALSE /\ unsafePublication = FALSE + /\ unsafeRetirement = FALSE +AcquireRecoveryForeground == + /\ recovery \in {"idle", "crashed"} /\ foreground = "none" + /\ foreground' = "recovery" /\ recovery' = "foreground" + /\ selected' = IF intent THEN selected ELSE version + /\ UNCHANGED <> +AcquireRecoveryEngine == + /\ recovery = "foreground" /\ foreground = "recovery" /\ engine = "none" + /\ engine' = "recovery" + /\ recovery' = IF retired THEN "retired" + ELSE IF complete THEN "complete" + ELSE IF intent THEN "cleaning" ELSE "leased" + /\ UNCHANGED <> +WriteIntent == + /\ recovery = "leased" /\ LocksHeld /\ SelectionMatches /\ ~published + /\ intent' = TRUE /\ recovery' = "cleaning" + /\ UNCHANGED <> +\* One effect represents one exact non-volume resource cleanup. Commit is separate. +CleanupOne == + /\ recovery \in {"leased", "cleaning"} /\ LocksHeld /\ ~published + /\ progress < 2 /\ (EnforceIntent => intent) + /\ (EnforceSelection => SelectionMatches) + /\ progress' = progress + 1 /\ recovery' = "cleaning" + /\ unsafeCleanup' = (unsafeCleanup \/ ~intent \/ ~SelectionMatches) + /\ UNCHANGED <> +CommitCleanup == + /\ recovery = "cleaning" /\ LocksHeld /\ intent + /\ SelectionMatches /\ progress = 2 /\ ~published + /\ complete' = TRUE /\ recovery' = "complete" + /\ UNCHANGED <> +RetireAbsentPublisher == + /\ recovery \in {"leased", "cleaning", "complete"} /\ LocksHeld + /\ SelectionMatches /\ ~published + /\ (EnforceCompletion => (intent /\ complete /\ progress = 2)) + /\ retired' = TRUE /\ recovery' = "retired" + /\ unsafeRetirement' = (unsafeRetirement \/ ~intent \/ ~complete \/ progress # 2) + /\ UNCHANGED <> +ReleaseRecovery == + /\ recovery = "retired" /\ LocksHeld + /\ foreground' = "none" /\ engine' = "none" /\ recovery' = "done" + /\ UNCHANGED <> +\* A crash releases kernel locks while retaining every committed durable field. +CrashRecovery == + /\ recovery \in {"foreground", "leased", "cleaning", "complete", "retired"} + /\ ~crashed /\ recovery' = "crashed" /\ crashed' = TRUE + /\ foreground' = "none" /\ engine' = "none" + /\ UNCHANGED <> +\* A cooperating identity/guest-boot change cannot occur under the Engine lease. +ChangeInputs == + /\ engine = "none" /\ version = 1 /\ version' = 2 + /\ UNCHANGED <> +RefuseRecovery == + /\ recovery \in {"leased", "cleaning", "complete"} /\ LocksHeld + /\ (~SelectionMatches \/ published) + /\ recovery' = "refused" /\ foreground' = "none" /\ engine' = "none" + /\ UNCHANGED <> +AcquirePublisherForeground == + /\ publisher = "idle" /\ foreground = "none" + /\ publisher' = "foreground" /\ foreground' = "publisher" + /\ UNCHANGED <> +AcquirePublisherEngine == + /\ publisher = "published" /\ foreground = "publisher" /\ engine = "none" + /\ publisher' = "active" /\ engine' = "publisher" + /\ UNCHANGED <> +ReleasePublisherEngine == + /\ publisher = "active" /\ engine = "publisher" + /\ publisher' = "running" /\ engine' = "none" + /\ UNCHANGED <> +FreshAllowed == ~intent /\ ~published +RetiredAllowed == intent /\ complete /\ retired /\ ~published +\* Ordinary bind publishes while holding the foreground lock, BEFORE Engine admission. +\* Recovery also needs that foreground lock, so intent and publication cannot race. +Publish == + /\ publisher = "foreground" /\ foreground = "publisher" /\ engine = "none" + /\ (FreshAllowed \/ RetiredAllowed \/ (~EnforceBarrier /\ ~published)) + /\ published' = TRUE /\ publisher' = "published" + /\ unsafePublication' = (unsafePublication \/ (intent /\ ~retired)) + /\ UNCHANGED <> +RefusePublisher == + /\ publisher = "foreground" /\ ~FreshAllowed /\ ~RetiredAllowed + /\ publisher' = "refused" /\ foreground' = "none" + /\ UNCHANGED <> +PublisherExit == + /\ publisher \in {"published", "active", "running"} /\ foreground = "publisher" + /\ publisher' = "exited" /\ foreground' = "none" /\ engine' = "none" + /\ UNCHANGED <> +Next == AcquireRecoveryForeground \/ AcquireRecoveryEngine \/ WriteIntent \/ CleanupOne + \/ CommitCleanup \/ RetireAbsentPublisher \/ ReleaseRecovery \/ CrashRecovery + \/ ChangeInputs \/ RefuseRecovery \/ AcquirePublisherForeground + \/ AcquirePublisherEngine \/ ReleasePublisherEngine \/ Publish \/ RefusePublisher + \/ PublisherExit +Spec == Init /\ [][Next]_vars +TypeOK == /\ foreground \in {"none", "recovery", "publisher"} + /\ engine \in {"none", "recovery", "publisher"} + /\ recovery \in {"idle", "foreground", "leased", "cleaning", "complete", + "retired", "crashed", "done", "refused"} + /\ publisher \in {"idle", "foreground", "published", "active", "running", + "refused", "exited"} + /\ intent \in BOOLEAN /\ progress \in 0..2 /\ complete \in BOOLEAN + /\ retired \in BOOLEAN /\ published \in BOOLEAN + /\ version \in 1..2 /\ selected \in 0..2 /\ crashed \in BOOLEAN + /\ unsafeCleanup \in BOOLEAN /\ unsafePublication \in BOOLEAN + /\ unsafeRetirement \in BOOLEAN +ForegroundBeforeEngine == engine # "none" => engine = foreground +NoUnwitnessedCleanup == ~unsafeCleanup +NoPrematurePublication == ~unsafePublication +NoUnconfirmedRetirement == ~unsafeRetirement +CompletionRequiresCleanup == complete => (intent /\ progress = 2) +======================================================================= diff --git a/tests/models/tla/absent-publication-recovery/negative.cfg b/tests/models/tla/absent-publication-recovery/negative.cfg new file mode 100644 index 000000000..16ebac56b --- /dev/null +++ b/tests/models/tla/absent-publication-recovery/negative.cfg @@ -0,0 +1,12 @@ +CONSTANTS EnforceIntent = TRUE + EnforceBarrier = FALSE + EnforceSelection = TRUE + EnforceCompletion = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK + ForegroundBeforeEngine + NoUnwitnessedCleanup + NoPrematurePublication + NoUnconfirmedRetirement + CompletionRequiresCleanup +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/absent-publication-recovery/positive.cfg b/tests/models/tla/absent-publication-recovery/positive.cfg new file mode 100644 index 000000000..db5e420e8 --- /dev/null +++ b/tests/models/tla/absent-publication-recovery/positive.cfg @@ -0,0 +1,12 @@ +CONSTANTS EnforceIntent = TRUE + EnforceBarrier = TRUE + EnforceSelection = TRUE + EnforceCompletion = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK + ForegroundBeforeEngine + NoUnwitnessedCleanup + NoPrematurePublication + NoUnconfirmedRetirement + CompletionRequiresCleanup +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/absent-publication-recovery/stale-selection.cfg b/tests/models/tla/absent-publication-recovery/stale-selection.cfg new file mode 100644 index 000000000..47cec1236 --- /dev/null +++ b/tests/models/tla/absent-publication-recovery/stale-selection.cfg @@ -0,0 +1,12 @@ +CONSTANTS EnforceIntent = TRUE + EnforceBarrier = TRUE + EnforceSelection = FALSE + EnforceCompletion = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK + ForegroundBeforeEngine + NoUnwitnessedCleanup + NoPrematurePublication + NoUnconfirmedRetirement + CompletionRequiresCleanup +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/absent-publication-recovery/unconfirmed-retirement.cfg b/tests/models/tla/absent-publication-recovery/unconfirmed-retirement.cfg new file mode 100644 index 000000000..7e0438c70 --- /dev/null +++ b/tests/models/tla/absent-publication-recovery/unconfirmed-retirement.cfg @@ -0,0 +1,12 @@ +CONSTANTS EnforceIntent = TRUE + EnforceBarrier = TRUE + EnforceSelection = TRUE + EnforceCompletion = FALSE +SPECIFICATION Spec +INVARIANTS TypeOK + ForegroundBeforeEngine + NoUnwitnessedCleanup + NoPrematurePublication + NoUnconfirmedRetirement + CompletionRequiresCleanup +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/absent-publication-recovery/unwitnessed-cleanup.cfg b/tests/models/tla/absent-publication-recovery/unwitnessed-cleanup.cfg new file mode 100644 index 000000000..07f3f71d4 --- /dev/null +++ b/tests/models/tla/absent-publication-recovery/unwitnessed-cleanup.cfg @@ -0,0 +1,12 @@ +CONSTANTS EnforceIntent = FALSE + EnforceBarrier = TRUE + EnforceSelection = TRUE + EnforceCompletion = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK + ForegroundBeforeEngine + NoUnwitnessedCleanup + NoPrematurePublication + NoUnconfirmedRetirement + CompletionRequiresCleanup +CHECK_DEADLOCK FALSE From b3a92467cbbfafa626558755f40fc6e3bdf7a5dc Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 03:18:18 -0400 Subject: [PATCH 06/25] fix(runtime): recover retained graphs after lost reboot publications --- docs/guides/native-candidate.md | 29 + packages/runtime-core/src/graph_cli.rs | 81 ++ packages/runtime-core/src/main.rs | 2 + .../graph/absent_publication_cleanup.rs | 1272 +++++++++++++++++ .../src/provider/graph/bridges/cleanup.rs | 198 ++- .../src/provider/graph/cleanup_enrollment.rs | 1 + .../src/provider/graph/dead_owner_cleanup.rs | 2 +- .../provider/graph/foreground/native_test.rs | 1 + .../absent_publication_recovery.rs | 500 +++++++ .../provider/graph/foreground/transport.rs | 3 + .../src/provider/graph/host_pin_recovery.rs | 8 +- .../src/provider/graph/host_relay.rs | 68 +- .../runtime-core/src/provider/graph/mod.rs | 59 +- .../tests/absent_recovery_cli_contract.rs | 107 ++ 14 files changed, 2320 insertions(+), 11 deletions(-) create mode 100644 packages/runtime-core/src/provider/graph/absent_publication_cleanup.rs create mode 100644 packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs create mode 100644 packages/runtime-core/tests/absent_recovery_cli_contract.rs diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 010e75fff..083e8b35a 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -200,6 +200,35 @@ directory itself during reboot, this command refuses: the old pin receipts no longer exist, and absent pathnames cannot stand in for their recorded owner identities. That case requires a separate selected absence-recovery procedure. +When a **physical host reboot** removed both the deterministic foreground +publication root and this run's relay-control root, inspect the distinct +absence path with the private original provider Owner and its exact +pre-migration host-filesystem inspection: + +```sh +./hack-native --candidate-root /absolute/private/candidate-home graph inspect-absent-publication-cleanup --run-id --original-owner-file --host-inspection-file --json +./hack-native --candidate-root /absolute/private/candidate-home graph recover-absent-publication-cleanup --run-id --original-owner-file --host-inspection-file --expect-selection --retain-data --accept-unpinned-post-reboot --json +``` + +Inspection does not create a publication root. A failed selected action can +leave only its private lock reservation; inspection recognizes that exact +lock-only state. Recovery locks the foreground publication before acquiring +the VM lease, then durably records the selected absence before any cleanup. +It retains volumes, verifies the stopped receipt, and records a separate +absent-publisher retirement. Ordinary publication and cleanup cannot infer +ownership from missing paths. The old foreground PID and original physical +volume are not proved by legacy graph receipts; this path requires explicit +acceptance of that post-reboot limitation and refuses a changed boot, present +or foreign publication, stale selection, pending state, or changed resources. +An interrupted recovery can resume only its exact selected intent on the same +host and immediate guest boot. A later successful restore treats this witness +as history; it never grants cleanup of the new generation. + +This operation does not change historical shared-source device identity. +Source-mounted projects require the separate selected source-continuity step +before normal same-run restore. The command's stopped/data-retained result is +not proof of application startup or routing. + Compatibility is qualified for specific bundle hashes and state formats. The displayed version alone does not establish frontend/executor or downgrade compatibility. V4 and candidate homes remain separate; this procedure does not diff --git a/packages/runtime-core/src/graph_cli.rs b/packages/runtime-core/src/graph_cli.rs index b93a6872a..d962c917f 100644 --- a/packages/runtime-core/src/graph_cli.rs +++ b/packages/runtime-core/src/graph_cli.rs @@ -69,6 +69,87 @@ pub fn command(candidate: &Candidate, args: &[&str]) -> Result (run, original, inspection), + _ => return Err(invalid()), + }; + #[cfg(target_os = "macos")] + { + return graph::inspect_absent_publication_cleanup( + candidate, + run, + Path::new(original), + Path::new(inspection), + ); + } + #[cfg(not(target_os = "macos"))] + { + let _ = (run, original, inspection); + return Err(invalid()); + } + } + if *action == "recover-absent-publication-cleanup" { + let (run, original, inspection, expected) = match *args { + [ + "--run-id", + run, + "--original-owner-file", + original, + "--host-inspection-file", + inspection, + "--expect-selection", + expected, + "--retain-data", + "--accept-unpinned-post-reboot", + ] + | [ + "--run-id", + run, + "--original-owner-file", + original, + "--host-inspection-file", + inspection, + "--expect-selection", + expected, + "--retain-data", + "--accept-unpinned-post-reboot", + "--json", + ] => (run, original, inspection, expected), + _ => return Err(invalid()), + }; + #[cfg(target_os = "macos")] + { + return graph::recover_absent_publication_cleanup( + candidate, + run, + expected, + Path::new(original), + Path::new(inspection), + ); + } + #[cfg(not(target_os = "macos"))] + { + let _ = (run, original, inspection, expected); + return Err(invalid()); + } + } if ["run-selection", "run-service"].contains(action) { return one_off::command(candidate, action, args); } diff --git a/packages/runtime-core/src/main.rs b/packages/runtime-core/src/main.rs index 5878fae07..558aff288 100644 --- a/packages/runtime-core/src/main.rs +++ b/packages/runtime-core/src/main.rs @@ -29,6 +29,8 @@ Usage: hack-local graph retire-recovered-publisher --run-id <32-hex> --expect-owner <32-hex> [--json] hack-local graph inspect-host-pin-recovery --run-id <32-hex> [--json] hack-local graph recover-host-pins --run-id <32-hex> --expect-selection <64-hex> --accept-legacy-device-rebind [--json] + hack-local graph inspect-absent-publication-cleanup --run-id <32-hex> --original-owner-file --host-inspection-file [--json] + hack-local graph recover-absent-publication-cleanup --run-id <32-hex> --original-owner-file --host-inspection-file --expect-selection <64-hex> --retain-data --accept-unpinned-post-reboot [--json] hack-local graph logs --run-id <32-hex> --service [--tail <1..1000>] [--json] hack-local graph exec --run-id <32-hex> --service [--workdir /path] [--timeout-seconds <1..120>] [--json] -- [args...] hack-local graph dependency-plan --dependencies [--json] diff --git a/packages/runtime-core/src/provider/graph/absent_publication_cleanup.rs b/packages/runtime-core/src/provider/graph/absent_publication_cleanup.rs new file mode 100644 index 000000000..ead3beec2 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/absent_publication_cleanup.rs @@ -0,0 +1,1272 @@ +//! Explicit data-retaining recovery when both ephemeral host publications were +//! lost across a physical host reboot. Missing paths never create ordinary +//! ownership authority: the selected, durable witness is the only admission. +use super::{ + Candidate, CandidateError, Engine, Kind, Receipt, dead_owner_cleanup, dependency_slots, + directory, foreground, host_pin_recovery, host_relay, initializer_cache, inspect_resource, + load, startup, state, +}; +use crate::provider::{ + ProjectShareIntent, artifact, host_pin::DeviceRebind, identity, lifecycle, state::Owner, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + fs, + os::unix::fs::MetadataExt, + path::{Path, PathBuf}, +}; + +const INTENT: &str = "absent-publication-cleanup.json"; +const RETIREMENT: &str = "absent-publication-retirement.json"; +const LIMIT: u64 = 2 * 1024 * 1024; + +fn refused() -> CandidateError { + CandidateError::new( + "graph_absent_publication_recovery", + "Selected post-reboot publication absence or retained graph identity changed; evidence and data were preserved.", + ) +} +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} +fn absent(path: &Path) -> Result { + match fs::symlink_metadata(path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true), + Ok(_) => Ok(false), + Err(_) => Err(refused()), + } +} +fn require_absent(path: &Path) -> Result<(), CandidateError> { + if !absent(path)? { + return Err(refused()); + } + Ok(()) +} +fn no_pending(root: &Path) -> Result<(), CandidateError> { + for name in [ + "state.pending", + "dead-owner-cleanup.pending", + "relay-cleanup-bridges.pending", + "one-off.pending", + "one-off-normalization.pending", + "absent-publication-cleanup.pending", + "absent-publication-retirement.pending", + ] { + require_absent(&root.join(name))?; + } + Ok(()) +} +fn retain_interrupted_publications(root: &Path) -> Result<(), CandidateError> { + super::journal::retain_file( + root, + "absent-publication-cleanup.pending", + "absent-publication-cleanup-recovery", + 4 * 1024 * 1024, + )?; + super::journal::retain_file( + root, + "absent-publication-retirement.pending", + "absent-publication-retirement-recovery", + 65536, + )?; + Ok(()) +} +fn private_input(path: &Path, limit: u64) -> Result, CandidateError> { + if !path.is_absolute() || fs::canonicalize(path).map_err(|_| refused())? != path { + return Err(refused()); + } + host_pin_recovery::read_raw(path, limit).map_err(|_| refused()) +} +fn device_only_share( + old: Option<&ProjectShareIntent>, + current: Option<&ProjectShareIntent>, + old_device: u64, + new_device: u64, +) -> bool { + match (old, current) { + (None, None) => true, + (Some(previous), Some(current)) => { + let mut expected = previous.clone(); + expected.device = new_device; + previous.device == old_device && *current == expected + } + _ => false, + } +} + +/// This is the exact serialization shape of the pre-migration inspection. It +/// binds the raw original Owner, one common host-device change and the boot +/// used to approve it; it is not a physical-volume UUID proof. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct FilesystemInspection { + schema: String, + selection_sha256: String, + owner_sha256: String, + machine: String, + host_boot_micros: u64, + provider_start_micros: u64, + old_device: u64, + new_device: u64, + pool_inode: u64, + storage: identity::DiskIdentity, + overlay: identity::DiskIdentity, + project_share: Option, + qualification: String, +} +impl FilesystemInspection { + fn verify(&self, owner_bytes: &[u8], old: &Owner) -> Result<(), CandidateError> { + let mut unsigned = self.clone(); + unsigned.selection_sha256.clear(); + if self.schema != "hack.host-filesystem-recovery/v1" + || self.qualification != "explicit-legacy-migration-original-volume-continuity-unproven" + || self.owner_sha256 != digest(owner_bytes) + || self.selection_sha256 + != digest(&serde_json::to_vec(&unsigned).map_err(|_| refused())?) + || self.machine != old.machine + || self.provider_start_micros != old.process.as_ref().ok_or_else(refused)?.start_micros + || self.old_device == self.new_device + || self.storage.device != self.new_device + || self.overlay.device != self.new_device + || old.storage.as_ref().is_none_or(|disk| { + disk.device != self.old_device + || disk.inode != self.storage.inode + || disk.bytes != self.storage.bytes + || disk.uuid != self.storage.uuid + }) + || old.overlay.as_ref().is_none_or(|disk| { + disk.device != self.old_device + || disk.inode != self.overlay.inode + || disk.bytes != self.overlay.bytes + || disk.uuid != self.overlay.uuid + }) + || !device_only_share( + old.project_share.as_ref(), + self.project_share.as_ref(), + self.old_device, + self.new_device, + ) + { + return Err(refused()); + } + Ok(()) + } +} + +#[cfg(all(test, feature = "environment-launcher"))] +pub(in crate::provider::graph) fn fixture_inspection( + owner_bytes: &[u8], + current: &Owner, + old_device: u64, + host_boot_micros: u64, + pool_inode: u64, +) -> Vec { + let old: Owner = serde_json::from_slice(owner_bytes).unwrap(); + let mut selection = FilesystemInspection { + schema: "hack.host-filesystem-recovery/v1".into(), + selection_sha256: String::new(), + owner_sha256: digest(owner_bytes), + machine: old.machine.clone(), + host_boot_micros, + provider_start_micros: old.process.as_ref().unwrap().start_micros, + old_device, + new_device: current.storage.as_ref().unwrap().device, + pool_inode, + storage: current.storage.clone().unwrap(), + overlay: current.overlay.clone().unwrap(), + project_share: current.project_share.clone(), + qualification: "explicit-legacy-migration-original-volume-continuity-unproven".into(), + }; + selection.selection_sha256 = digest(&serde_json::to_vec(&selection).unwrap()); + serde_json::to_vec_pretty(&selection).unwrap() +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Selection { + version: u8, + candidate: PathBuf, + run: String, + owner: String, + namespace: String, + plan: String, + original_owner_path: PathBuf, + original_owner_sha256: String, + filesystem_inspection_path: PathBuf, + filesystem_inspection_sha256: String, + current_owner_sha256: String, + graph_sha256: String, + host_boot_micros: u64, + old_device: u64, + new_device: u64, + previous_guest_boot: String, + current_guest_boot: String, + foreground_root: PathBuf, + control_root: PathBuf, + source_shared: Option, + environment_inventory: Value, + scoped_bridge_projection: Value, + retained_volumes: BTreeMap, + dependency_reservation: Option, + qualification: String, +} +impl Selection { + pub(super) fn digest(&self) -> Result { + Ok(digest(&serde_json::to_vec(self).map_err(|_| refused())?)) + } + pub(super) fn matches_graph(&self, receipt: &Receipt) -> bool { + self.run == receipt.run + && self.owner == receipt.owner + && self.namespace == receipt.namespace + && self.plan == receipt.plan_id + && self.source_shared == receipt.source.as_ref().and_then(|s| s.shared.clone()) + } + pub(super) fn previous_boot(&self) -> &str { + &self.previous_guest_boot + } + pub(super) fn control_root(&self) -> &Path { + &self.control_root + } +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Intent { + version: u8, + selection_sha256: String, + selection: Selection, + original: Receipt, + environment: Value, + bridges: super::bridges::cleanup::Selection, + #[serde(default, skip_serializing_if = "Option::is_none")] + prior_bridges: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + complete_sha256: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Retirement { + version: u8, + selection_sha256: String, + complete_sha256: String, + owner: String, + run: String, +} + +fn selected_volumes( + engine: &Engine<'_>, + receipt: &Receipt, +) -> Result, CandidateError> { + let mut volumes = BTreeMap::new(); + for (key, resource) in receipt + .resources + .iter() + .filter(|(_, resource)| resource.kind == Kind::Volume) + { + let actual = inspect_resource(engine, receipt, resource)?.ok_or_else(refused)?; + let created = actual["CreatedAt"] + .as_str() + .filter(|value| !value.is_empty()); + if created.is_none() { + return Err(refused()); + } + volumes.insert( + key.clone(), + json!({ + "expected_name": resource.name, + "observed_name": actual["Name"], + "observed_created_at": actual["CreatedAt"], + "observed_mountpoint": actual["Mountpoint"], + "observed_driver": actual["Driver"], + "observed_labels_sha256": digest( + &serde_json::to_vec(&actual["Labels"]).map_err(|_| refused())? + ), + "labels": super::expected_labels(receipt, resource), + }), + ); + } + Ok(volumes) +} + +fn current_owner( + candidate: &Candidate, + old: &Owner, + inspection: &FilesystemInspection, + engine: &Engine<'_>, +) -> Result<(Owner, String), CandidateError> { + lifecycle::host_filesystem::no_auxiliary_update(candidate)?; + if lifecycle::host_filesystem::host_boot_micros()? != inspection.host_boot_micros { + return Err(refused()); + } + let old_process = old.process.as_ref().ok_or_else(refused)?; + // SAFETY: geteuid has no arguments or side effects. + identity::verify( + old_process, + old_process, + &artifact::root(candidate).join("smolvm-bin"), + unsafe { libc::geteuid() }, + ) + .map_err(|_| refused())?; + DeviceRebind { + old: inspection.old_device, + current: inspection.new_device, + } + .definitely_dead_before_boot(old_process, inspection.host_boot_micros) + .map_err(|_| refused())?; + let owner = Owner::load(candidate)?; + lifecycle::verify_disks(candidate, &owner)?; + host_pin_recovery::verify_guest_identity(engine)?; + let pool = + fs::symlink_metadata(candidate.state_root.join("run/smolvm")).map_err(|_| refused())?; + if old.phase != "running" + || owner.phase != "running" + || old.token != owner.token + || owner.token != engine.guest().incarnation() + || old.guest_boot_id.as_deref() != owner.previous_guest_boot_id.as_deref() + || owner.guest_boot_id.as_deref() != Some(engine.guest().boot_id()) + || old.guest_boot_id.as_deref() == owner.guest_boot_id.as_deref() + || owner.storage.as_ref() != Some(&inspection.storage) + || owner.overlay.as_ref() != Some(&inspection.overlay) + || owner.project_share != inspection.project_share + || pool.ino() != inspection.pool_inode + || pool.dev() != inspection.new_device + { + return Err(refused()); + } + let mut expected = old.clone(); + expected.storage = owner.storage.clone(); + expected.overlay = owner.overlay.clone(); + expected.project_share = owner.project_share.clone(); + expected.process = owner.process.clone(); + expected.guest_boot_id = owner.guest_boot_id.clone(); + expected.previous_guest_boot_id = owner.previous_guest_boot_id.clone(); + expected.daemon_pid = owner.daemon_pid; + expected.daemon_start = owner.daemon_start; + if expected != owner { + return Err(refused()); + } + let owner_bytes = host_pin_recovery::read_raw( + &candidate.state_root.join("run/smolvm/owner.json"), + 1024 * 1024, + )?; + let raw_owner: Owner = serde_json::from_slice(&owner_bytes).map_err(|_| refused())?; + if raw_owner != owner { + return Err(refused()); + } + let owner_sha256 = digest(&owner_bytes); + Ok((owner, owner_sha256)) +} + +fn publication_roots(selection: &Selection, allow_lock_only: bool) -> Result<(), CandidateError> { + require_absent(&selection.control_root)?; + match fs::symlink_metadata(&selection.foreground_root) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound && !allow_lock_only => Ok(()), + Ok(_) if allow_lock_only => { + state::check_private_directory(&selection.foreground_root)?; + let mut entries = fs::read_dir(&selection.foreground_root).map_err(|_| refused())?; + let entry = entries.next().ok_or_else(refused)?.map_err(|_| refused())?; + if entry.file_name() != "operation.lock" || entries.next().is_some() { + return Err(refused()); + } + let lock = fs::symlink_metadata(entry.path()).map_err(|_| refused())?; + // SAFETY: geteuid has no arguments or side effects. + if !lock.is_file() + || lock.nlink() != 1 + || lock.mode() & 0o7777 != 0o600 + || lock.uid() != unsafe { libc::geteuid() } + { + return Err(refused()); + } + Ok(()) + } + _ => Err(refused()), + } +} + +fn select_content( + candidate: &Candidate, + run: &str, + original_owner_path: &Path, + filesystem_inspection_path: &Path, + engine: &Engine<'_>, + allow_lock_only: bool, +) -> Result<(Selection, Receipt), CandidateError> { + let (receipt, root) = load(candidate, engine, run)?; + no_pending(&root)?; + if receipt.phase != "ready-observed" + || receipt.relay_cleanup.is_some() + || receipt.relay_startup.is_none() + || !super::hex(&receipt.owner, 32) + { + return Err(refused()); + } + initializer_cache::require_resolved(&receipt)?; + startup::require_dependency_rebind_complete(&root, &receipt)?; + dead_owner_cleanup::require_historical_recovery(&root, &receipt)?; + let old_bytes = private_input(original_owner_path, 1024 * 1024)?; + let old: Owner = serde_json::from_slice(&old_bytes).map_err(|_| refused())?; + let inspection_bytes = private_input(filesystem_inspection_path, 64 * 1024)?; + let inspection: FilesystemInspection = + serde_json::from_slice(&inspection_bytes).map_err(|_| refused())?; + inspection.verify(&old_bytes, &old)?; + let (owner, current_owner_sha256) = current_owner(candidate, &old, &inspection, engine)?; + if old.checkout != candidate.checkout + || old.machine != inspection.machine + || owner.machine != inspection.machine + || old + .project_share + .as_ref() + .is_some_and(|share| share.device != inspection.old_device) + || receipt + .source + .as_ref() + .and_then(|source| source.shared.as_ref()) + != old.project_share.as_ref() + { + return Err(refused()); + } + let startup = receipt.relay_startup.as_ref().ok_or_else(refused)?; + let foreground_root = foreground::transport::root(candidate, run)?; + let graph_bytes = host_pin_recovery::read_raw(&root.join("state.json"), LIMIT)?; + if graph_bytes != serde_json::to_vec_pretty(&receipt).map_err(|_| refused())? { + return Err(refused()); + } + let graph_sha256 = digest(&graph_bytes); + let rebind = DeviceRebind { + old: inspection.old_device, + current: inspection.new_device, + }; + let dependency_reservation = + dependency_slots::inspect_legacy(candidate, run, rebind, inspection.host_boot_micros)?; + let previous_guest_boot = old.guest_boot_id.clone().ok_or_else(refused)?; + let selection = Selection { + version: 1, + candidate: candidate.checkout.clone(), + run: run.into(), + owner: receipt.owner.clone(), + namespace: receipt.namespace.clone(), + plan: receipt.plan_id.clone(), + original_owner_path: original_owner_path.to_path_buf(), + original_owner_sha256: digest(&old_bytes), + filesystem_inspection_path: filesystem_inspection_path.to_path_buf(), + filesystem_inspection_sha256: digest(&inspection_bytes), + current_owner_sha256, + graph_sha256, + host_boot_micros: inspection.host_boot_micros, + old_device: inspection.old_device, + new_device: inspection.new_device, + previous_guest_boot: previous_guest_boot.clone(), + current_guest_boot: owner.guest_boot_id.ok_or_else(refused)?, + foreground_root, + control_root: startup.control_root.clone(), + source_shared: receipt + .source + .as_ref() + .and_then(|source| source.shared.clone()), + environment_inventory: serde_json::to_value(super::environment::cleanup_inventory( + candidate, engine, &receipt, &root, + )?) + .map_err(|_| refused())?, + scoped_bridge_projection: super::bridges::cleanup::scoped_absence_projection( + candidate, + engine, + &receipt, + &previous_guest_boot, + )?, + retained_volumes: selected_volumes(engine, &receipt)?, + dependency_reservation, + qualification: "explicit-unpinned-post-host-reboot-original-volume-continuity-unproven" + .into(), + }; + publication_roots(&selection, allow_lock_only)?; + host_pin_recovery::verify_volume_projections(engine, &receipt, &selection.retained_volumes)?; + for resource in receipt.resources.values() { + if resource.kind != Kind::Volume && inspect_resource(engine, &receipt, resource)?.is_none() + { + return Err(refused()); + } + } + Ok((selection, receipt)) +} + +pub fn inspect( + candidate: &Candidate, + run: &str, + original_owner_path: &Path, + filesystem_inspection_path: &Path, +) -> Result { + let existing = Reservation::inspect_existing(candidate, run)?; + let engine = Engine::connect_cleanup_wait(candidate)?; + let (selected, _) = select_content( + candidate, + run, + original_owner_path, + filesystem_inspection_path, + &engine, + existing.is_some(), + )?; + if let Some(reservation) = &existing { + reservation.verify()?; + } + Ok(json!({ + "run": run, + "selection_sha256": selected.digest()?, + "host_boot_micros": selected.host_boot_micros, + "qualification": selected.qualification, + "data_retained": true, + })) +} + +struct Reservation { + root: PathBuf, + identity: (u64, u64), + lock: state::Lock, +} +impl Reservation { + fn inspect_existing(candidate: &Candidate, run: &str) -> Result, CandidateError> { + let root = foreground::transport::root(candidate, run)?; + match fs::symlink_metadata(&root) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Ok(_) => { + state::check_private_directory(&root)?; + let lock = state::Lock::acquire_existing(&root)?; + let metadata = fs::symlink_metadata(&root).map_err(|_| refused())?; + let value = Self { + root, + identity: (metadata.dev(), metadata.ino()), + lock, + }; + value.verify()?; + Ok(Some(value)) + } + Err(_) => Err(refused()), + } + } + fn acquire(candidate: &Candidate, run: &str) -> Result { + let root = foreground::transport::root(candidate, run)?; + let lock = match fs::symlink_metadata(&root) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + state::Lock::acquire(&root)? + } + Ok(_) => { + state::check_private_directory(&root)?; + state::Lock::acquire_existing(&root)? + } + Err(_) => return Err(refused()), + }; + let metadata = fs::symlink_metadata(&root).map_err(|_| refused())?; + let value = Self { + root, + identity: (metadata.dev(), metadata.ino()), + lock, + }; + value.verify()?; + Ok(value) + } + fn verify(&self) -> Result<(), CandidateError> { + let metadata = fs::symlink_metadata(&self.root).map_err(|_| refused())?; + if !metadata.is_dir() + || (metadata.dev(), metadata.ino()) != self.identity + || state::check_private_directory(&self.root).is_err() + { + return Err(refused()); + } + host_pin_recovery::exact_lock_path(&self.root, &self.lock).map_err(|_| refused())?; + let lock = fs::symlink_metadata(self.root.join("operation.lock")).map_err(|_| refused())?; + // SAFETY: geteuid has no arguments or side effects. + if lock.mode() & 0o7777 != 0o600 + || lock.uid() != unsafe { libc::geteuid() } + || lock.nlink() != 1 + { + return Err(refused()); + } + // This verifier is intentionally limited to the reserved root; it + // does not infer control-root or graph ownership from the lock. + let mut entries = fs::read_dir(&self.root).map_err(|_| refused())?; + let entry = entries.next().ok_or_else(refused)?.map_err(|_| refused())?; + if entry.file_name() != "operation.lock" || entries.next().is_some() { + return Err(refused()); + } + Ok(()) + } +} + +fn verify_static_with( + candidate: &Candidate, + run: &str, + engine: &Engine<'_>, + selected: &Selection, + original: &Receipt, + foreground_root: &Path, + verify_lock: impl Fn() -> Result<(), CandidateError>, +) -> Result { + if selected.version != 1 + || selected.candidate != candidate.checkout + || selected.run != run + || selected.foreground_root != foreground_root + || selected.qualification + != "explicit-unpinned-post-host-reboot-original-volume-continuity-unproven" + || !selected.matches_graph(original) + || selected.digest()?.len() != 64 + { + return Err(refused()); + } + verify_lock()?; + publication_roots(selected, true)?; + lifecycle::host_filesystem::no_auxiliary_update(candidate)?; + if lifecycle::host_filesystem::host_boot_micros()? != selected.host_boot_micros { + return Err(refused()); + } + let old_bytes = private_input(&selected.original_owner_path, 1024 * 1024)?; + let inspection_bytes = private_input(&selected.filesystem_inspection_path, 64 * 1024)?; + if digest(&old_bytes) != selected.original_owner_sha256 + || digest(&inspection_bytes) != selected.filesystem_inspection_sha256 + { + return Err(refused()); + } + let old: Owner = serde_json::from_slice(&old_bytes).map_err(|_| refused())?; + let inspection: FilesystemInspection = + serde_json::from_slice(&inspection_bytes).map_err(|_| refused())?; + inspection.verify(&old_bytes, &old)?; + let (owner, owner_sha256) = current_owner(candidate, &old, &inspection, engine)?; + if owner_sha256 != selected.current_owner_sha256 + || owner.token != selected.owner + || owner.guest_boot_id.as_deref() != Some(&selected.current_guest_boot) + || owner.previous_guest_boot_id.as_deref() != Some(&selected.previous_guest_boot) + || inspection.old_device != selected.old_device + || inspection.new_device != selected.new_device + { + return Err(refused()); + } + let (receipt, root) = load(candidate, engine, run)?; + no_pending(&root)?; + let receipt_bytes = host_pin_recovery::read_raw(&root.join("state.json"), LIMIT)?; + if receipt_bytes != serde_json::to_vec_pretty(&receipt).map_err(|_| refused())? { + return Err(refused()); + } + if dead_owner_cleanup::immutable(&receipt)? != dead_owner_cleanup::immutable(original)? + || !selected.matches_graph(&receipt) + || !["ready-observed", "cleanup-intent", "stopped-data-retained"] + .contains(&receipt.phase.as_str()) + { + return Err(refused()); + } + if receipt.phase == "ready-observed" && digest(&receipt_bytes) != selected.graph_sha256 { + return Err(refused()); + } + if receipt.phase == "ready-observed" + && (serde_json::to_value(super::environment::cleanup_inventory( + candidate, engine, &receipt, &root, + )?) + .map_err(|_| refused())? + != selected.environment_inventory + || super::bridges::cleanup::scoped_absence_projection( + candidate, + engine, + &receipt, + &selected.previous_guest_boot, + )? != selected.scoped_bridge_projection) + { + return Err(refused()); + } + host_pin_recovery::verify_volume_projections(engine, &receipt, &selected.retained_volumes)?; + let rebind = DeviceRebind { + old: selected.old_device, + current: selected.new_device, + }; + if let Some(legacy) = &selected.dependency_reservation { + dependency_slots::verify_legacy_remaining( + candidate, + run, + rebind, + legacy, + receipt.phase != "ready-observed", + )?; + } else if dependency_slots::inspect_legacy(candidate, run, rebind, selected.host_boot_micros)? + .is_some() + { + return Err(refused()); + } + verify_lock()?; + Ok(receipt) +} + +fn verify_static( + candidate: &Candidate, + run: &str, + engine: &Engine<'_>, + selected: &Selection, + original: &Receipt, + reservation: &Reservation, +) -> Result { + verify_static_with( + candidate, + run, + engine, + selected, + original, + &reservation.root, + || reservation.verify(), + ) +} + +fn read_intent(root: &Path) -> Result, CandidateError> { + let path = root.join(INTENT); + if absent(&path)? { + return Ok(None); + } + let intent: Intent = state::read_bounded(&path, 4 * 1024 * 1024).map_err(|_| refused())?; + if intent.version != 1 + || intent.selection_sha256 != intent.selection.digest()? + || intent.selection.graph_sha256 + != digest(&serde_json::to_vec_pretty(&intent.original).map_err(|_| refused())?) + || !intent.selection.matches_graph(&intent.original) + { + return Err(refused()); + } + Ok(Some(intent)) +} +fn completed_receipt( + candidate: &Candidate, + engine: &Engine<'_>, + intent: &Intent, + current: &Receipt, + root: &Path, +) -> Result { + if current.phase != "stopped-data-retained" { + return Err(refused()); + } + let environment = super::environment::cleanup_inventory(candidate, engine, current, root)?; + if serde_json::to_value(&environment).map_err(|_| refused())? != intent.environment { + return Err(refused()); + } + super::bridges::cleanup::verify_recovery_file( + root, + &intent.bridges, + intent.prior_bridges.as_ref(), + )?; + host_relay::inspect_cleanup_absence( + candidate, + engine, + current, + &environment, + &intent.bridges, + &intent.selection, + )?; + Ok(digest(&host_pin_recovery::read_raw( + &root.join("state.json"), + LIMIT, + )?)) +} + +fn retire( + candidate: &Candidate, + engine: &Engine<'_>, + intent: &Intent, + current: &Receipt, + root: &Path, + reservation: &Reservation, +) -> Result<(), CandidateError> { + reservation.verify()?; + let complete = intent.complete_sha256.as_deref().ok_or_else(refused)?; + if completed_receipt(candidate, engine, intent, current, root)? != complete { + return Err(refused()); + } + let retirement = Retirement { + version: 1, + selection_sha256: intent.selection_sha256.clone(), + complete_sha256: complete.into(), + owner: intent.selection.owner.clone(), + run: intent.selection.run.clone(), + }; + let path = root.join(RETIREMENT); + if absent(&path)? { + verify_static( + candidate, + &intent.selection.run, + engine, + &intent.selection, + &intent.original, + reservation, + )?; + state::write(&path, &retirement)?; + } else { + let recorded: Retirement = state::read_bounded(&path, 65536).map_err(|_| refused())?; + if recorded.version != retirement.version + || recorded.selection_sha256 != retirement.selection_sha256 + || recorded.complete_sha256 != retirement.complete_sha256 + || recorded.owner != retirement.owner + || recorded.run != retirement.run + { + return Err(refused()); + } + } + reservation.verify() +} + +/// A pending absence intent blocks ordinary publication. Completed retirement +/// permits only the explicit restored-generation bind; it is not a Pin. +pub(super) fn publication_allowed( + candidate: &Candidate, + run: &str, + retired: bool, +) -> Result<(), CandidateError> { + let root = directory(candidate, run)?; + require_absent(&root.join("absent-publication-cleanup.pending"))?; + require_absent(&root.join("absent-publication-retirement.pending"))?; + if absent(&root.join(INTENT))? { + return Ok(()); + } + if !retired { + return Err(refused()); + } + let intent = read_intent(&root)?.ok_or_else(refused)?; + let complete = intent.complete_sha256.as_deref().ok_or_else(refused)?; + let retirement: Retirement = + state::read_bounded(&root.join(RETIREMENT), 65536).map_err(|_| refused())?; + let current: Receipt = state::read_bounded(&root.join("state.json"), LIMIT)?; + if retirement.version != 1 + || retirement.selection_sha256 != intent.selection_sha256 + || retirement.complete_sha256 != complete + || retirement.owner != current.owner + || retirement.run != run + || current.phase != "stopped-data-retained" + { + return Err(refused()); + } + let current_sha256 = digest(&host_pin_recovery::read_raw( + &root.join("state.json"), + LIMIT, + )?); + if current_sha256 != complete && retained(&root, ¤t)? { + return Err(refused()); + } + Ok(()) +} + +/// Retention is bound to the completed stopped receipt, not to missing paths. +pub(super) fn retained(root: &Path, receipt: &Receipt) -> Result { + let Some(intent) = read_intent(root)? else { + return Ok(false); + }; + let Some(complete) = intent.complete_sha256.as_deref() else { + return Err(refused()); + }; + let retirement: Retirement = + state::read_bounded(&root.join(RETIREMENT), 65536).map_err(|_| refused())?; + let current: Receipt = state::read_bounded(&root.join("state.json"), LIMIT)?; + let current_sha256 = digest(&host_pin_recovery::read_raw( + &root.join("state.json"), + LIMIT, + )?); + let requested_sha256 = digest(&serde_json::to_vec_pretty(receipt).map_err(|_| refused())?); + if current_sha256 != requested_sha256 + || receipt.phase != "stopped-data-retained" + || retirement.version != 1 + || retirement.selection_sha256 != intent.selection_sha256 + || retirement.complete_sha256 != complete + || retirement.run != receipt.run + || retirement.owner != receipt.owner + || serde_json::to_vec_pretty(¤t).map_err(|_| refused())? + != serde_json::to_vec_pretty(receipt).map_err(|_| refused())? + { + return Err(refused()); + } + if requested_sha256 != complete { + if intent.original.run != receipt.run + || intent.original.owner != receipt.owner + || intent.original.namespace != receipt.namespace + || !super::restore_history::confirms_prior_generation(root, receipt)? + { + return Err(refused()); + } + super::cleanup_enrollment::retention_receipt(receipt, false)?; + return Ok(false); + } + Ok(true) +} + +/// Exact completed, retired first-generation proof for a later independently +/// selected source-continuity transition. The proof grants no authority for a +/// subsequent graph generation and does not change the original graph source. +#[derive(Clone, Serialize)] +#[allow(dead_code)] // Consumed by the independently selected source-continuity unit. +pub(super) struct CompletedSourceProof { + pub original_ready_sha256: String, + pub completed_stopped_sha256: String, + pub intent_raw_sha256: String, + pub retirement_raw_sha256: String, + pub original_owner_sha256: String, + pub current_owner_sha256: String, + pub host_boot_micros: u64, + pub previous_guest_boot: String, + pub current_guest_boot: String, + pub old_share: Option, + pub current_share: Option, + pub retained_volumes: BTreeMap, +} + +/// The caller holds the foreground retirement lock before its Engine lease and +/// must recheck this proof at the final publication boundary. A historical +/// sidecar with a different stopped receipt cannot lend authority. +#[allow(dead_code)] // Exposed for the subsequent source-continuity unit. +pub(super) fn verify_completed_under( + candidate: &Candidate, + engine: &Engine<'_>, + run: &str, + receipt: &Receipt, + retired: &foreground::transport::Retired, +) -> Result, CandidateError> { + let root = directory(candidate, run)?; + if absent(&root.join(INTENT))? { + return Ok(None); + } + retired.verify()?; + let intent = read_intent(&root)?.ok_or_else(refused)?; + let complete = intent.complete_sha256.as_deref().ok_or_else(refused)?; + let (current, _) = load(candidate, engine, run)?; + if current.phase != "stopped-data-retained" + || digest(&host_pin_recovery::read_raw( + &root.join("state.json"), + LIMIT, + )?) != complete + || serde_json::to_vec_pretty(¤t).map_err(|_| refused())? + != serde_json::to_vec_pretty(receipt).map_err(|_| refused())? + { + return Err(refused()); + } + let foreground_root = foreground::transport::root(candidate, run)?; + verify_static_with( + candidate, + run, + engine, + &intent.selection, + &intent.original, + &foreground_root, + || retired.verify(), + )?; + if completed_receipt(candidate, engine, &intent, ¤t, &root)? != complete { + return Err(refused()); + } + let retirement_bytes = host_pin_recovery::read_raw(&root.join(RETIREMENT), 65536)?; + let retirement: Retirement = + serde_json::from_slice(&retirement_bytes).map_err(|_| refused())?; + if retirement.version != 1 + || retirement.selection_sha256 != intent.selection_sha256 + || retirement.complete_sha256 != complete + || retirement.run != run + || retirement.owner != current.owner + { + return Err(refused()); + } + let old_bytes = private_input(&intent.selection.original_owner_path, 1024 * 1024)?; + let old: Owner = serde_json::from_slice(&old_bytes).map_err(|_| refused())?; + let inspection_bytes = private_input(&intent.selection.filesystem_inspection_path, 64 * 1024)?; + let inspection: FilesystemInspection = + serde_json::from_slice(&inspection_bytes).map_err(|_| refused())?; + inspection.verify(&old_bytes, &old)?; + retired.verify()?; + Ok(Some(CompletedSourceProof { + original_ready_sha256: intent.selection.graph_sha256.clone(), + completed_stopped_sha256: complete.into(), + intent_raw_sha256: digest(&host_pin_recovery::read_raw( + &root.join(INTENT), + 4 * 1024 * 1024, + )?), + retirement_raw_sha256: digest(&retirement_bytes), + original_owner_sha256: intent.selection.original_owner_sha256.clone(), + current_owner_sha256: intent.selection.current_owner_sha256.clone(), + host_boot_micros: intent.selection.host_boot_micros, + previous_guest_boot: intent.selection.previous_guest_boot.clone(), + current_guest_boot: intent.selection.current_guest_boot.clone(), + old_share: old.project_share, + current_share: inspection.project_share, + retained_volumes: intent.selection.retained_volumes.clone(), + })) +} + +/// Explicit, hash-selected cleanup and separate durable absent-publisher +/// retirement. This command never removes a selected graph volume. +pub fn recover( + candidate: &Candidate, + run: &str, + expected: &str, + original_owner_path: &Path, + filesystem_inspection_path: &Path, +) -> Result { + if !super::hex(expected, 64) { + return Err(refused()); + } + // Foreground publication takes this lock before the VM operation lease. + // Following the same order prevents a new publisher from racing admission. + let reservation = Reservation::acquire(candidate, run)?; + let engine = Engine::connect_cleanup_wait(candidate)?; + let root = directory(candidate, run)?; + retain_interrupted_publications(&root)?; + let mut intent = if let Some(existing) = read_intent(&root)? { + if existing.selection_sha256 != expected + || existing.selection.original_owner_path != original_owner_path + || existing.selection.filesystem_inspection_path != filesystem_inspection_path + { + return Err(refused()); + } + existing + } else { + let (selection, original) = select_content( + candidate, + run, + original_owner_path, + filesystem_inspection_path, + &engine, + true, + )?; + if selection.digest()? != expected { + return Err(refused()); + } + let environment = + super::environment::cleanup_inventory(candidate, &engine, &original, &root)?; + let environment_value = serde_json::to_value(&environment).map_err(|_| refused())?; + if environment_value != selection.environment_inventory { + return Err(refused()); + } + host_relay::cleanup_preflight(&engine, &original, &root, false)?; + let bridges = super::bridges::cleanup::capture_previous_boot_absence( + candidate, &engine, &original, &selection, + )?; + if super::bridges::cleanup::selection_absence_projection(&bridges)? + != selection.scoped_bridge_projection + { + return Err(refused()); + } + let prior_bridges = + super::bridges::cleanup::capture_prior_generation(&root, &bridges, &original)?; + if prior_bridges.is_some() + && !super::restore_history::confirms_prior_generation(&root, &original)? + { + return Err(refused()); + } + let selected = Intent { + version: 1, + selection_sha256: expected.into(), + selection, + original, + environment: environment_value, + bridges, + prior_bridges, + complete_sha256: None, + }; + reservation.verify()?; + state::write(&root.join(INTENT), &selected)?; + #[cfg(test)] + super::fault_pause(&root, run, "absent-after-intent")?; + selected + }; + let current = verify_static( + candidate, + run, + &engine, + &intent.selection, + &intent.original, + &reservation, + )?; + if intent.complete_sha256.is_none() { + if current.phase == "stopped-data-retained" { + // Cleanup may have completed immediately before the intent's final + // commit. Exact confirmation permits a deterministic retry. + intent.complete_sha256 = Some(completed_receipt( + candidate, &engine, &intent, ¤t, &root, + )?); + state::write(&root.join(INTENT), &intent)?; + } else { + super::bridges::cleanup::verify_remaining_absence( + candidate, + &engine, + ¤t, + &intent.bridges, + &intent.selection, + )?; + super::bridges::cleanup::verify_recovery_file( + &root, + &intent.bridges, + intent.prior_bridges.as_ref(), + )?; + super::bridges::cleanup::recover_persist(&root, &intent.bridges)?; + reservation.verify()?; + let cleaned = super::cleanup_owned_fenced( + candidate, + &engine, + current, + &root, + false, + true, + || { + verify_static( + candidate, + run, + &engine, + &intent.selection, + &intent.original, + &reservation, + ) + .map(|_| ()) + }, + )?; + reservation.verify()?; + intent.complete_sha256 = Some(completed_receipt( + candidate, &engine, &intent, &cleaned, &root, + )?); + state::write(&root.join(INTENT), &intent)?; + } + } + let (current, _) = load(candidate, &engine, run)?; + let complete = completed_receipt(candidate, &engine, &intent, ¤t, &root)?; + if intent.complete_sha256.as_deref() != Some(complete.as_str()) { + return Err(refused()); + } + if let Some(reservation_record) = &intent.selection.dependency_reservation { + dependency_slots::recover_cleaned( + candidate, + ¤t, + Some(( + DeviceRebind { + old: intent.selection.old_device, + current: intent.selection.new_device, + }, + reservation_record, + )), + )?; + } else { + dependency_slots::recover_cleaned(candidate, ¤t, None)?; + } + retire(candidate, &engine, &intent, ¤t, &root, &reservation)?; + Ok(json!({ + "run": run, + "phase": "stopped-data-retained", + "publisher_retired": true, + "data_retained": true, + "selection_sha256": expected, + "qualification": intent.selection.qualification, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + fn candidate() -> (super::super::tests::Fixture, Candidate) { + let fixture = super::super::tests::Fixture::new(); + let candidate = Candidate::discover(&fixture.0).unwrap(); + (fixture, candidate) + } + + #[test] + fn read_only_inspection_of_existing_reservation_never_creates_a_root() { + let (_fixture, candidate) = candidate(); + let run = "a".repeat(32); + let root = foreground::transport::root(&candidate, &run).unwrap(); + assert!( + Reservation::inspect_existing(&candidate, &run) + .unwrap() + .is_none() + ); + assert!(absent(&root).unwrap()); + let acquired = Reservation::acquire(&candidate, &run).unwrap(); + acquired.verify().unwrap(); + drop(acquired); + let inspected = Reservation::inspect_existing(&candidate, &run) + .unwrap() + .unwrap(); + inspected.verify().unwrap(); + assert_eq!( + fs::read_dir(&root).unwrap().count(), + 1, + "only the lock-only reservation is present" + ); + drop(inspected); + fs::remove_file(root.join("operation.lock")).unwrap(); + fs::remove_dir(root).unwrap(); + } + + #[test] + fn substituted_foreground_lock_path_refuses_without_deleting_replacement() { + let (_fixture, candidate) = candidate(); + let run = "b".repeat(32); + let acquired = Reservation::acquire(&candidate, &run).unwrap(); + let root = acquired.root.clone(); + let path = acquired.root.join("operation.lock"); + let saved = acquired.root.join("operation.held"); + fs::rename(&path, &saved).unwrap(); + fs::write(&path, b"foreign replacement").unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + let replacement = fs::symlink_metadata(&path).unwrap(); + assert!(acquired.verify().is_err()); + let after = fs::symlink_metadata(&path).unwrap(); + assert_eq!( + (after.dev(), after.ino()), + (replacement.dev(), replacement.ino()) + ); + assert_eq!(fs::read(&path).unwrap(), b"foreign replacement"); + drop(acquired); + fs::remove_file(&path).unwrap(); + fs::rename(saved, &path).unwrap(); + fs::remove_file(&path).unwrap(); + fs::remove_dir(&root).unwrap(); + } + + #[test] + fn interrupted_intent_blocks_every_ordinary_publisher_mode() { + let (_fixture, candidate) = candidate(); + let run = "c".repeat(32); + let root = directory(&candidate, &run).unwrap(); + state::private_directory(&root).unwrap(); + fs::write(root.join("absent-publication-cleanup.pending"), b"partial").unwrap(); + assert!(publication_allowed(&candidate, &run, false).is_err()); + assert!(publication_allowed(&candidate, &run, true).is_err()); + assert!(absent(&root.join(INTENT)).unwrap()); + } + + #[test] + fn legacy_share_rebind_refuses_every_non_device_change() { + let old = ProjectShareIntent { + project: PathBuf::from("/private/tmp/selected"), + guest_path: "/workspace".into(), + device: 10, + inode: 92, + unfiltered_source: true, + }; + let mut current = old.clone(); + current.device = 11; + assert!(device_only_share(Some(&old), Some(¤t), 10, 11)); + assert!(!device_only_share(None, Some(¤t), 10, 11)); + assert!(!device_only_share(Some(&old), None, 10, 11)); + assert!(!device_only_share(Some(&old), Some(¤t), 9, 11)); + for changed in [ + { + let mut value = current.clone(); + value.project = PathBuf::from("/private/tmp/other"); + value + }, + { + let mut value = current.clone(); + value.guest_path = "/different".into(); + value + }, + { + let mut value = current.clone(); + value.inode += 1; + value + }, + { + let mut value = current.clone(); + value.unfiltered_source = false; + value + }, + ] { + assert!(!device_only_share(Some(&old), Some(&changed), 10, 11)); + } + } +} diff --git a/packages/runtime-core/src/provider/graph/bridges/cleanup.rs b/packages/runtime-core/src/provider/graph/bridges/cleanup.rs index c76005ce1..17779815f 100644 --- a/packages/runtime-core/src/provider/graph/bridges/cleanup.rs +++ b/packages/runtime-core/src/provider/graph/bridges/cleanup.rs @@ -11,12 +11,21 @@ pub(crate) struct Selection { previous_boot: Option, #[serde(default, skip_serializing_if = "Option::is_none")] predecessor_owner: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + predecessor_absence: Option, run: String, plan: String, capacity: u8, serial: u64, selected: BTreeMap, } +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct AbsentPredecessor { + selection_sha256: String, + previous_boot: String, + control_root: PathBuf, +} #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -54,6 +63,15 @@ fn validate_selection_recovery( receipt: &Receipt, selection: &Selection, host_pin: Option<&super::super::host_pin_recovery::Witness>, +) -> Result<(), CandidateError> { + validate_selection_with_absence(engine, receipt, selection, host_pin, None) +} +fn validate_selection_with_absence( + engine: &Engine<'_>, + receipt: &Receipt, + selection: &Selection, + host_pin: Option<&super::super::host_pin_recovery::Witness>, + absence: Option<&super::super::absent_publication_cleanup::Selection>, ) -> Result<(), CandidateError> { let capacity = engine.guest().bridge_intent().map_or(0, |v| v.slots); if selection.version != 1 @@ -81,6 +99,24 @@ fn validate_selection_recovery( return Err(invalid()); } } + match (&selection.predecessor_absence, absence) { + (None, None) => {} + (None, Some(witness)) + if !selection.selected.is_empty() + && selection.predecessor_owner.is_none() + && selection.previous_boot.as_deref() == Some(witness.previous_boot()) + && witness.matches_graph(receipt) => {} + (Some(recorded), Some(witness)) + if recorded.selection_sha256 == witness.digest()? + && recorded.previous_boot == witness.previous_boot() + && selection.previous_boot.as_deref() == Some(witness.previous_boot()) + && recorded.control_root == witness.control_root() + && witness.matches_graph(receipt) + && selection.selected.is_empty() + && selection.predecessor_owner.is_none() + && pinned_predecessor_eligible(receipt) => {} + _ => return Err(invalid()), + } Ok(()) } @@ -92,7 +128,9 @@ fn validate_bindings( if let Some(previous) = &selection.previous_boot { if previous == &selection.boot || previous_boot != Some(previous.as_str()) - || (selection.selected.is_empty() && selection.predecessor_owner.is_none()) + || (selection.selected.is_empty() + && selection.predecessor_owner.is_none() + && selection.predecessor_absence.is_none()) { return Err(invalid()); } @@ -105,6 +143,23 @@ fn validate_bindings( }) { return Err(invalid()); } + if selection + .predecessor_absence + .as_ref() + .is_some_and(|absence| { + !hex(&absence.selection_sha256, 64) + || absence.previous_boot != selection.previous_boot.as_deref().unwrap_or_default() + || receipt + .relay_startup + .as_ref() + .is_none_or(|startup| startup.control_root != absence.control_root) + || selection.predecessor_owner.is_some() + || !selection.selected.is_empty() + || !pinned_predecessor_eligible(receipt) + }) + { + return Err(invalid()); + } let store = Store { version: 1, owner: selection.owner.clone(), @@ -153,6 +208,7 @@ pub(crate) fn capture( boot: engine.guest().boot_id().into(), previous_boot: None, predecessor_owner: None, + predecessor_absence: None, run: receipt.run.clone(), plan: receipt.plan_id.clone(), capacity: engine.guest().bridge_intent().map_or(0, |v| v.slots), @@ -248,6 +304,7 @@ pub(crate) fn capture_previous_boot( boot: engine.guest().boot_id().into(), previous_boot: Some(previous.into()), predecessor_owner: None, + predecessor_absence: None, run: receipt.run.clone(), plan: receipt.plan_id.clone(), capacity: engine.guest().bridge_intent().map_or(0, |v| v.slots), @@ -280,6 +337,7 @@ pub(crate) fn capture_previous_boot( || prior.boot != previous || prior.previous_boot.is_some() || prior.predecessor_owner.is_some() + || prior.predecessor_absence.is_some() || prior.run != selection.run || prior.plan != selection.plan || prior.capacity != selection.capacity @@ -310,6 +368,106 @@ pub(crate) fn capture_previous_boot( Ok(selection) } +/// A missing relay-control root has no Pin fingerprint. Select a distinct, +/// witness-bound predecessor only with the exact current bridge registry. +pub(crate) fn capture_previous_boot_absence( + candidate: &Candidate, + engine: &Engine<'_>, + receipt: &Receipt, + witness: &super::super::absent_publication_cleanup::Selection, +) -> Result { + let store = strict_store(candidate, engine)?; + let mut selection = Selection { + version: 1, + owner: engine.guest().incarnation().into(), + boot: engine.guest().boot_id().into(), + previous_boot: Some(witness.previous_boot().into()), + predecessor_owner: None, + predecessor_absence: None, + run: receipt.run.clone(), + plan: receipt.plan_id.clone(), + capacity: engine + .guest() + .bridge_intent() + .map_or(0, |value| value.slots), + serial: store.next_launch_serial, + selected: store + .slots + .into_iter() + .filter(|(_, assignment)| assignment.run == receipt.run) + .map(|(slot, assignment)| { + ( + slot, + Selected { + assignment, + helper: None, + }, + ) + }) + .collect(), + }; + if selection.selected.is_empty() { + selection.predecessor_absence = Some(AbsentPredecessor { + selection_sha256: witness.digest()?, + previous_boot: witness.previous_boot().into(), + control_root: witness.control_root().to_path_buf(), + }); + } + validate_selection_with_absence(engine, receipt, &selection, None, Some(witness))?; + Ok(selection) +} + +/// Selection binds only this run's assignments. Independent sibling +/// allocations may advance the global serial without changing its authority. +pub(crate) fn scoped_absence_projection( + candidate: &Candidate, + engine: &Engine<'_>, + receipt: &Receipt, + previous_boot: &str, +) -> Result { + let store = strict_store(candidate, engine)?; + let selected = store + .slots + .iter() + .filter(|(_, assignment)| assignment.run == receipt.run) + .collect::>(); + Ok(json!({ + "owner": engine.guest().incarnation(), + "current_boot": engine.guest().boot_id(), + "previous_boot": previous_boot, + "run": receipt.run, + "plan": receipt.plan_id, + "capacity": engine.guest().bridge_intent().map_or(0, |value| value.slots), + "selected": selected, + })) +} + +pub(crate) fn selection_absence_projection(selection: &Selection) -> Result { + Ok(json!({ + "owner": selection.owner, + "current_boot": selection.boot, + "previous_boot": selection.previous_boot, + "run": selection.run, + "plan": selection.plan, + "capacity": selection.capacity, + "selected": selection.selected.iter() + .map(|(slot, selected)| (slot, &selected.assignment)) + .collect::>(), + })) +} + +pub(crate) fn verify_remaining_absence( + candidate: &Candidate, + engine: &Engine<'_>, + receipt: &Receipt, + selection: &Selection, + witness: &super::super::absent_publication_cleanup::Selection, +) -> Result<(), CandidateError> { + validate_selection_with_absence(engine, receipt, selection, None, Some(witness))?; + let store = strict_store(candidate, engine)?; + remaining_matches(&store, receipt, selection) +} + /// Pending recovery may resume only the exact remaining reservations. Released /// slots may be absent; a replacement or new reservation is never adopted. pub(crate) fn verify_remaining_recovery( @@ -344,6 +502,7 @@ pub(crate) fn verify_live_remaining( fn live_remaining_matches(store: &Store, selection: &Selection) -> Result<(), CandidateError> { if selection.previous_boot.is_some() || selection.predecessor_owner.is_some() + || selection.predecessor_absence.is_some() || store.owner != selection.owner || store.next_launch_serial < selection.serial { @@ -414,6 +573,7 @@ fn prior_generation( && prior.boot == current.previous_boot.as_deref().unwrap_or_default() && prior.previous_boot.is_none() && prior.predecessor_owner.is_none() + && prior.predecessor_absence.is_none() && prior.run == current.run && prior.plan == current.plan && prior.capacity == current.capacity @@ -436,6 +596,7 @@ fn prior_generation( && prior.boot == current.previous_boot.as_deref().unwrap_or_default() && prior.previous_boot.is_none() && prior.predecessor_owner.is_none() + && prior.predecessor_absence.is_none() && prior.run == current.run && prior.plan == current.plan && prior.capacity == current.capacity @@ -575,6 +736,16 @@ pub(crate) fn read_recovery( validate_selection_recovery(engine, receipt, &selection, host_pin)?; Ok(selection) } +pub(crate) fn read_recovery_absence( + engine: &Engine<'_>, + receipt: &Receipt, + root: &std::path::Path, + witness: &super::super::absent_publication_cleanup::Selection, +) -> Result { + let selection = state::read_bounded(&selection_path(root)?, 65536)?; + validate_selection_with_absence(engine, receipt, &selection, None, Some(witness))?; + Ok(selection) +} pub(crate) fn verify_recovery( candidate: &Candidate, @@ -598,6 +769,30 @@ pub(crate) fn verify_recovery( engine.guest().verify()?; Ok(()) } +pub(crate) fn verify_recovery_absence( + candidate: &Candidate, + engine: &Engine<'_>, + receipt: &Receipt, + selection: &Selection, + witness: &super::super::absent_publication_cleanup::Selection, +) -> Result<(), CandidateError> { + validate_selection_with_absence(engine, receipt, selection, None, Some(witness))?; + let store = strict_store(candidate, engine)?; + if store.next_launch_serial < selection.serial + || store + .slots + .values() + .any(|assignment| assignment.run == receipt.run) + { + return Err(invalid()); + } + for (slot, selected) in &selection.selected { + if let Some(helper) = &selected.helper { + relay::verify_cleanup(engine, *slot, &selected.assignment, helper)?; + } + } + engine.guest().verify() +} #[cfg(test)] pub(crate) fn verify( @@ -622,6 +817,7 @@ mod tests { boot: "boot".into(), previous_boot: None, predecessor_owner: None, + predecessor_absence: None, run: "b".repeat(32), plan: "c".repeat(64), capacity: 0, diff --git a/packages/runtime-core/src/provider/graph/cleanup_enrollment.rs b/packages/runtime-core/src/provider/graph/cleanup_enrollment.rs index 6c187f2e0..665db1b7b 100644 --- a/packages/runtime-core/src/provider/graph/cleanup_enrollment.rs +++ b/packages/runtime-core/src/provider/graph/cleanup_enrollment.rs @@ -119,6 +119,7 @@ pub(super) fn retention(root: &Path, receipt: &Receipt) -> Result<(), CandidateE #[cfg(target_os = "macos")] if super::live_owner_cleanup::retained(root, receipt)? || super::dead_owner_cleanup::retained(root, receipt)? + || super::absent_publication_cleanup::retained(root, receipt)? { return Ok(()); } diff --git a/packages/runtime-core/src/provider/graph/dead_owner_cleanup.rs b/packages/runtime-core/src/provider/graph/dead_owner_cleanup.rs index 9e32a865f..14d5599b4 100644 --- a/packages/runtime-core/src/provider/graph/dead_owner_cleanup.rs +++ b/packages/runtime-core/src/provider/graph/dead_owner_cleanup.rs @@ -38,7 +38,7 @@ fn selected(receipt: &Receipt) -> Result { &serde_json::to_vec_pretty(receipt).map_err(|_| refused())?, )) } -fn immutable(receipt: &Receipt) -> Result { +pub(super) fn immutable(receipt: &Receipt) -> Result { let mut value = serde_json::to_value(receipt).map_err(|_| refused())?; value.as_object_mut().ok_or_else(refused)?.remove("phase"); for group in ["resources", "probes"] { diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test.rs b/packages/runtime-core/src/provider/graph/foreground/native_test.rs index aba7ac312..84665fc9e 100644 --- a/packages/runtime-core/src/provider/graph/foreground/native_test.rs +++ b/packages/runtime-core/src/provider/graph/foreground/native_test.rs @@ -542,6 +542,7 @@ mod dependency_rebind; mod dependency_slots; mod startup_cancellation; +mod absent_publication_recovery; mod host_pin_recovery; mod retired_recovery_cleanup; mod same_boot_recovery; diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs b/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs new file mode 100644 index 000000000..aa290d1d8 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs @@ -0,0 +1,500 @@ +//! Owned capacity-two VM qualification for absent post-host-reboot publication. +//! The host reboot evidence is explicitly simulated in this isolated fixture; +//! this does not prove physical-volume continuity on a real reboot. +use super::dependency_rebind::{checked_cli, exec, snapshot}; +use super::*; +use crate::provider::{lifecycle, state::Owner}; +use std::os::unix::fs::MetadataExt; + +fn ready(owner: &mut Process, run: &str, deadline: Instant) { + loop { + assert!( + owner.poll().is_none(), + "foreground owner exited before ready" + ); + if let Some(end) = owner.out.iter().position(|byte| *byte == b'\n') { + let value: Value = serde_json::from_slice(&owner.out[..end]).unwrap(); + assert_eq!(value["kind"], "graph_foreground_ready"); + assert_eq!(value["run"], run); + return; + } + assert!(Instant::now() < deadline, "foreground readiness deadline"); + std::thread::sleep(Duration::from_millis(10)); + } +} + +fn refused_cli( + binary: &Path, + candidate: &Candidate, + args: &[&str], + expected_code: &str, + deadline: Instant, +) { + let mut process = Process::start(binary, candidate, args, None); + let status = process.wait(deadline); + assert_eq!( + status.code(), + Some(2), + "expected a CLI refusal, not a crash" + ); + assert!(process.out.is_empty(), "refusal emitted a result"); + let error: Value = serde_json::from_slice(&process.err).expect("structured CLI error"); + assert_eq!(error["code"], expected_code); +} + +fn inspect_args<'a>(run: &'a str, old: &'a str, prior: &'a str) -> [&'a str; 9] { + [ + "graph", + "inspect-absent-publication-cleanup", + "--run-id", + run, + "--original-owner-file", + old, + "--host-inspection-file", + prior, + "--json", + ] +} +fn recover_args<'a>(run: &'a str, old: &'a str, prior: &'a str, hash: &'a str) -> [&'a str; 13] { + [ + "graph", + "recover-absent-publication-cleanup", + "--run-id", + run, + "--original-owner-file", + old, + "--host-inspection-file", + prior, + "--expect-selection", + hash, + "--retain-data", + "--accept-unpinned-post-reboot", + "--json", + ] +} + +/// Scoped fixture file mutation restores only bytes it wrote, including on a +/// panic. It never replaces managed provider or graph state. +struct PrivateBytes<'a> { + path: &'a Path, + original: Vec, + substituted: Vec, +} +impl<'a> PrivateBytes<'a> { + fn replace(path: &'a Path, substituted: Vec) -> Self { + let original = fs::read(path).unwrap(); + fs::write(path, &substituted).unwrap(); + Self { + path, + original, + substituted, + } + } +} +impl Drop for PrivateBytes<'_> { + fn drop(&mut self) { + if fs::read(self.path).ok().as_deref() == Some(self.substituted.as_slice()) { + fs::write(self.path, &self.original).unwrap(); + } + } +} + +struct Moved { + original: PathBuf, + moved: PathBuf, +} +impl Moved { + fn new(original: PathBuf) -> Self { + let moved = original.with_extension("absent-fixture-held"); + assert!(!moved.exists()); + fs::rename(&original, &moved).unwrap(); + Self { original, moved } + } +} +impl Drop for Moved { + fn drop(&mut self) { + if self.moved.exists() { + assert!( + !self.original.exists(), + "foreign path replaced selected fixture" + ); + fs::rename(&self.moved, &self.original).unwrap(); + } + } +} + +struct FaultChild(Option); +impl Drop for FaultChild { + fn drop(&mut self) { + if let Some(mut child) = self.0.take() { + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +#[test] +#[ignore = "Only the parent owned VM fixture starts this exact fault helper"] +fn absent_publication_after_bridge_fault_child() { + let candidate = + Candidate::discover(Path::new(&std::env::var("HACK_LOCAL_TEST_ROOT").unwrap())).unwrap(); + graph::absent_publication_cleanup::recover( + &candidate, + &std::env::var("HACK_LOCAL_GRAPH_RUN").unwrap(), + &std::env::var("HACK_LOCAL_GRAPH_SELECTION").unwrap(), + Path::new(&std::env::var("HACK_LOCAL_GRAPH_OLD_OWNER").unwrap()), + Path::new(&std::env::var("HACK_LOCAL_GRAPH_INSPECTION").unwrap()), + ) + .unwrap(); +} + +#[test] +#[ignore = "Owned capacity-two native VM, pinned image and external 300s watchdog required"] +fn explicit_absent_publication_cleanup_retains_selected_volume_and_sibling() { + let deadline = Instant::now() + Duration::from_secs(270); + let candidate = + Candidate::discover(Path::new(&std::env::var("HACK_LOCAL_TEST_ROOT").unwrap())).unwrap(); + let binary = PathBuf::from(std::env::var("HACK_LOCAL_TEST_BINARY").unwrap()); + let image = std::env::var("HACK_LOCAL_TEST_IMAGE").unwrap(); + let fixtures = [graph::tests::Fixture::new(), graph::tests::Fixture::new()]; + let runs = [ + graph::probes::token().unwrap(), + graph::probes::token().unwrap(), + ]; + let private = graph::tests::Fixture::new(); + let mut plans = Vec::new(); + let mut selections = Vec::new(); + let mut dependencies = Vec::new(); + for (index, fixture) in fixtures.iter().enumerate() { + state::write(&fixture.0.join("compose.yaml"), &json!({ + "services":{"web":{"image":image,"read_only":true,"network_mode":"none", + "init":true,"user":"0:0","entrypoint":["/bin/sleep","300"],"command":[], + "volumes":["data:/data"],"healthcheck":{"test":["CMD","/bin/hack-graph-startup-app","complete"], + "interval":"200ms","timeout":"2s","retries":10,"start_period":"500ms"}}}, + "volumes":{"data":{}} + })).unwrap(); + let review = project::plan( + &candidate, + project::PlanOptions { + branch: None, + project: &fixture.0, + compose_file: Path::new("compose.yaml"), + profiles: &[], + }, + ) + .unwrap(); + let selection = private.0.join(format!("dependencies-{index}.json")); + state::write(&selection, &json!({"version":1,"plan":review.plan_id, + "artifact":"/tmp/unused-control-only-artifact","artifact_sha256":"a".repeat(64),"dependencies":[]})).unwrap(); + let dependency = checked_cli( + "dependency-plan", + &binary, + &candidate, + &[ + "graph", + "dependency-plan", + "--dependencies", + selection.to_str().unwrap(), + "--json", + ], + deadline, + ); + plans.push(review); + selections.push(selection); + dependencies.push(dependency); + } + let start = |index: usize| { + let normalized = fixtures[index].0.join("compose.yaml"); + Process::start( + &binary, + &candidate, + &[ + "graph", + "serve", + "--project", + fixtures[index].0.to_str().unwrap(), + "--file", + "compose.yaml", + "--expect-plan", + &plans[index].plan_id, + "--run-id", + &runs[index], + "--ready", + "web=healthy", + "--timeout-seconds", + "90", + "--dependencies", + selections[index].to_str().unwrap(), + "--expect-dependencies", + dependencies[index]["dependency_plan_id"].as_str().unwrap(), + "--normalized-file", + normalized.to_str().unwrap(), + "--expect-original", + &plans[index].plan.compose_sha256, + "--expect-namespace", + &plans[index].plan.namespace, + "--json", + ], + None, + ) + }; + let mut selected_owner = start(0); + let mut selected_cleanup = Cleanup { + binary: &binary, + candidate: &candidate, + run: &runs[0], + done: false, + }; + ready(&mut selected_owner, &runs[0], deadline); + assert_eq!( + exec(&binary, &candidate, &runs[0], "web", "write-data", deadline)["exit_code"], + 0 + ); + let original = snapshot(&candidate, &runs[0], deadline).receipt; + assert_eq!(original.phase, "ready-observed"); + let old_owner = Owner::load(&candidate).unwrap(); + let old_boot = old_owner.guest_boot_id.clone().unwrap(); + selected_owner.child.kill().unwrap(); + selected_owner.wait(deadline); + assert_eq!(lifecycle::down(&candidate).unwrap().phase, "stopped"); + let new_boot = lifecycle::up(&candidate).unwrap().guest_boot_id.unwrap(); + assert_ne!(old_boot, new_boot); + + let mut sibling_owner = start(1); + let mut sibling_cleanup = Cleanup { + binary: &binary, + candidate: &candidate, + run: &runs[1], + done: false, + }; + ready(&mut sibling_owner, &runs[1], deadline); + assert_eq!( + exec(&binary, &candidate, &runs[1], "web", "write-data", deadline)["exit_code"], + 0 + ); + let sibling = serde_json::to_vec(&snapshot(&candidate, &runs[1], deadline).receipt).unwrap(); + + let current = Owner::load(&candidate).unwrap(); + let boot = lifecycle::host_filesystem::host_boot_micros().unwrap(); + let new_device = current.storage.as_ref().unwrap().device; + let old_device = new_device.checked_add(1).unwrap(); + let mut synthetic = old_owner.clone(); + synthetic.storage.as_mut().unwrap().device = old_device; + synthetic.overlay.as_mut().unwrap().device = old_device; + if let Some(share) = synthetic.project_share.as_mut() { + share.device = old_device; + } + let prior_process = synthetic.process.as_mut().unwrap(); + prior_process.pid = i32::MAX; + prior_process.start_micros = boot - 1; + let old_path = private.0.join("pre-host-boot-owner.json"); + state::write(&old_path, &synthetic).unwrap(); + let old_bytes = fs::read(&old_path).unwrap(); + let pool = fs::symlink_metadata(candidate.state_root.join("run/smolvm")).unwrap(); + let inspection_path = private.0.join("pre-migration-inspection.json"); + state::write( + &inspection_path, + &serde_json::from_slice::(&graph::absent_publication_cleanup::fixture_inspection( + &old_bytes, + ¤t, + old_device, + boot, + pool.ino(), + )) + .unwrap(), + ) + .unwrap(); + let graph_root = graph::directory(&candidate, &runs[0]).unwrap(); + let raw_graph = fs::read(graph_root.join("state.json")).unwrap(); + let publisher_root = transport::root(&candidate, &runs[0]).unwrap(); + let control_root = original + .relay_startup + .as_ref() + .unwrap() + .control_root + .clone(); + assert!(publisher_root.exists() && control_root.exists()); + fs::remove_dir_all(&publisher_root).unwrap(); + fs::remove_dir_all(&control_root).unwrap(); + + let old = old_path.to_str().unwrap(); + let prior = inspection_path.to_str().unwrap(); + let initial = checked_cli( + "inspect-absent-publications", + &binary, + &candidate, + &inspect_args(&runs[0], old, prior), + deadline, + ); + let selection = initial["selection_sha256"].as_str().unwrap(); + assert_eq!(selection.len(), 64); + let intent_path = graph_root.join("absent-publication-cleanup.json"); + refused_cli( + &binary, + &candidate, + &recover_args(&runs[0], old, prior, &"0".repeat(64)), + "graph_absent_publication_recovery", + deadline, + ); + assert!(!intent_path.exists()); + assert_eq!(fs::read(graph_root.join("state.json")).unwrap(), raw_graph); + let refreshed = checked_cli( + "inspect-after-stale-selection", + &binary, + &candidate, + &inspect_args(&runs[0], old, prior), + deadline, + ); + assert_eq!(refreshed["selection_sha256"], selection); + { + let mut whitespace = old_bytes.clone(); + whitespace.push(b' '); + let _changed = PrivateBytes::replace(&old_path, whitespace); + refused_cli( + &binary, + &candidate, + &recover_args(&runs[0], old, prior, selection), + "graph_absent_publication_recovery", + deadline, + ); + assert!(!intent_path.exists()); + } + { + let _missing = Moved::new(inspection_path.clone()); + refused_cli( + &binary, + &candidate, + &recover_args(&runs[0], old, prior, selection), + "graph_absent_publication_recovery", + deadline, + ); + assert!(!intent_path.exists()); + } + { + let state_path = graph_root.join("state.json"); + let mut changed = raw_graph.clone(); + changed.push(b' '); + let _changed = PrivateBytes::replace(&state_path, changed); + refused_cli( + &binary, + &candidate, + &recover_args(&runs[0], old, prior, selection), + "graph_absent_publication_recovery", + deadline, + ); + assert!(!intent_path.exists()); + } + { + let lock = publisher_root.join("operation.lock"); + let _original = Moved::new(lock.clone()); + fs::write(&lock, b"substituted lock path").unwrap(); + refused_cli( + &binary, + &candidate, + &recover_args(&runs[0], old, prior, selection), + "graph_absent_publication_recovery", + deadline, + ); + assert!(!intent_path.exists()); + fs::remove_file(&lock).unwrap(); + } + let child = Command::new(std::env::current_exe().unwrap()) + .args(["--ignored", "--exact", + "provider::graph::foreground::native_test::absent_publication_recovery::absent_publication_after_bridge_fault_child"]) + .env("HACK_LOCAL_TEST_ROOT", std::env::var("HACK_LOCAL_TEST_ROOT").unwrap()) + .env("HACK_LOCAL_GRAPH_RUN", &runs[0]) + .env("HACK_LOCAL_GRAPH_SELECTION", selection) + .env("HACK_LOCAL_GRAPH_OLD_OWNER", old) + .env("HACK_LOCAL_GRAPH_INSPECTION", prior) + .env("HACK_LOCAL_GRAPH_FAULT", "absent-after-bridge-release") + .stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()) + .spawn().unwrap(); + let mut fault = FaultChild(Some(child)); + let marker = graph_root.join("fault-absent-after-bridge-release.json"); + let fault_deadline = Instant::now() + Duration::from_secs(25); + while !marker.exists() { + assert!( + fault.0.as_mut().unwrap().try_wait().unwrap().is_none(), + "fault child exited before selected boundary" + ); + assert!( + Instant::now() < fault_deadline, + "selected fault boundary deadline" + ); + std::thread::sleep(Duration::from_millis(20)); + } + assert_eq!(state::read::(&marker).unwrap()["run"], runs[0]); + assert!( + intent_path.exists(), + "absence intent was durable before bridge release" + ); + assert_eq!( + state::read::(&graph_root.join("state.json")) + .unwrap() + .phase, + "cleanup-intent", + "graph intent was durable before bridge release" + ); + assert!( + graph::absent_publication_cleanup::publication_allowed(&candidate, &runs[0], false) + .is_err() + ); + assert!( + graph::absent_publication_cleanup::publication_allowed(&candidate, &runs[0], true).is_err() + ); + fault.0.as_mut().unwrap().kill().unwrap(); + fault.0.as_mut().unwrap().wait().unwrap(); + fault.0 = None; + let result = checked_cli( + "recover-absent-publications", + &binary, + &candidate, + &recover_args(&runs[0], old, prior, selection), + deadline, + ); + assert_eq!(result["phase"], "stopped-data-retained"); + assert_eq!(result["publisher_retired"], true); + let stopped = snapshot(&candidate, &runs[0], deadline).receipt; + assert_eq!( + stopped.resources["volume:data"].name, + original.resources["volume:data"].name + ); + let repeated = checked_cli( + "idempotent-absent-retirement", + &binary, + &candidate, + &recover_args(&runs[0], old, prior, selection), + deadline, + ); + assert_eq!(repeated["publisher_retired"], true); + assert!(sibling_owner.poll().is_none()); + assert_eq!( + serde_json::to_vec(&snapshot(&candidate, &runs[1], deadline).receipt).unwrap(), + sibling + ); + assert_eq!( + exec(&binary, &candidate, &runs[1], "web", "read-data", deadline)["exit_code"], + 0 + ); + + // Keep the selected retained volume for the later source-continuity + // fixture. The external harness tears down this isolated test pool. + selected_cleanup.done = true; + let removed = checked_cli( + "remove-sibling-data", + &binary, + &candidate, + &[ + "graph", + "cleanup", + "--run-id", + &runs[1], + "--remove-data", + "--json", + ], + deadline, + ); + assert_eq!(removed["phase"], "removed"); + assert!(sibling_owner.wait(deadline).success()); + sibling_cleanup.done = true; +} diff --git a/packages/runtime-core/src/provider/graph/foreground/transport.rs b/packages/runtime-core/src/provider/graph/foreground/transport.rs index 2064bfbc6..975cb8540 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport.rs @@ -772,6 +772,7 @@ impl Publication { Self::bind_mode(candidate, run, true) } fn bind_mode(candidate: &Candidate, run: &str, retired: bool) -> Result { + super::super::absent_publication_cleanup::publication_allowed(candidate, run, retired)?; let root = root(candidate, run)?; let lock = if retired { state::check_private_directory(&root).map_err(|_| refused())?; @@ -780,6 +781,8 @@ impl Publication { state::private_directory(&root).map_err(|_| refused())?; state::Lock::acquire(&root).map_err(|_| refused())? }; + // The foreground lock serializes a concurrent absence-intent writer. + super::super::absent_publication_cleanup::publication_allowed(candidate, run, retired)?; for name in ["control.sock", "owner.json"] { match fs::symlink_metadata(root.join(name)) { Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} diff --git a/packages/runtime-core/src/provider/graph/host_pin_recovery.rs b/packages/runtime-core/src/provider/graph/host_pin_recovery.rs index a41c94b31..1846c7018 100644 --- a/packages/runtime-core/src/provider/graph/host_pin_recovery.rs +++ b/packages/runtime-core/src/provider/graph/host_pin_recovery.rs @@ -40,7 +40,7 @@ fn digest(bytes: &[u8]) -> String { /// guest check so a changed receipt cannot impersonate the running kernel boot. /// execute_cleanup itself compares /proc boot ID and /storage owner under the /// held Engine/Guest lease before this fixed acknowledgement is emitted. -fn verify_guest_identity(engine: &Engine<'_>) -> Result<(), CandidateError> { +pub(super) fn verify_guest_identity(engine: &Engine<'_>) -> Result<(), CandidateError> { let acknowledged = engine .guest() .execute_cleanup("printf 'host-pin-guest-identity-v1\\n'", &[]) @@ -50,7 +50,7 @@ fn verify_guest_identity(engine: &Engine<'_>) -> Result<(), CandidateError> { } Ok(()) } -fn read_raw(path: &Path, limit: u64) -> Result, CandidateError> { +pub(super) fn read_raw(path: &Path, limit: u64) -> Result, CandidateError> { read_raw_with(path, limit, || {}) } fn read_raw_with( @@ -114,7 +114,7 @@ fn absent(path: &Path) -> Result { } } -fn exact_lock_path(root: &Path, held: &state::Lock) -> Result<(), CandidateError> { +pub(super) fn exact_lock_path(root: &Path, held: &state::Lock) -> Result<(), CandidateError> { let pathname = fs::symlink_metadata(root.join("operation.lock")).map_err(|_| refused())?; if !pathname.is_file() || pathname.nlink() != 1 @@ -258,7 +258,7 @@ pub(super) fn selected_for_old_publisher( Ok(Some(witness)) } -fn verify_volume_projections( +pub(super) fn verify_volume_projections( engine: &Engine<'_>, receipt: &Receipt, selected: &BTreeMap, diff --git a/packages/runtime-core/src/provider/graph/host_relay.rs b/packages/runtime-core/src/provider/graph/host_relay.rs index 49946e7f5..967b8076e 100644 --- a/packages/runtime-core/src/provider/graph/host_relay.rs +++ b/packages/runtime-core/src/provider/graph/host_relay.rs @@ -508,6 +508,47 @@ pub(super) fn inspect_cleanup_recovery( environment: &super::super::environment_recovery::GraphInventory, bridge_selection: &bridges::cleanup::Selection, host_pin: Option<&super::host_pin_recovery::Witness>, +) -> Result<[u8; 32], CandidateError> { + inspect_cleanup_selected( + candidate, + engine, + expected, + remove_data, + environment, + bridge_selection, + SelectedCleanupProof::Existing(host_pin), + ) +} +pub(super) fn inspect_cleanup_absence( + candidate: &Candidate, + engine: &Engine<'_>, + expected: &Receipt, + environment: &super::super::environment_recovery::GraphInventory, + bridge_selection: &bridges::cleanup::Selection, + witness: &super::absent_publication_cleanup::Selection, +) -> Result<[u8; 32], CandidateError> { + inspect_cleanup_selected( + candidate, + engine, + expected, + false, + environment, + bridge_selection, + SelectedCleanupProof::Absence(witness), + ) +} +enum SelectedCleanupProof<'a> { + Existing(Option<&'a super::host_pin_recovery::Witness>), + Absence(&'a super::absent_publication_cleanup::Selection), +} +fn inspect_cleanup_selected( + candidate: &Candidate, + engine: &Engine<'_>, + expected: &Receipt, + remove_data: bool, + environment: &super::super::environment_recovery::GraphInventory, + bridge_selection: &bridges::cleanup::Selection, + proof: SelectedCleanupProof<'_>, ) -> Result<[u8; 32], CandidateError> { let (receipt, root) = archive::load_confirmation(candidate, engine, &expected.run)?; match fs::symlink_metadata(root.join("state.pending")) { @@ -552,10 +593,33 @@ pub(super) fn inspect_cleanup_recovery( require_retained_volume(resource, value.is_some(), remove_data)?; observations.insert(key, value); } - if &bridges::cleanup::read_recovery(engine, &receipt, &root, host_pin)? != bridge_selection { + let recorded = match proof { + SelectedCleanupProof::Absence(witness) => { + bridges::cleanup::read_recovery_absence(engine, &receipt, &root, witness)? + } + SelectedCleanupProof::Existing(host_pin) => { + bridges::cleanup::read_recovery(engine, &receipt, &root, host_pin)? + } + }; + if &recorded != bridge_selection { return Err(refused()); } - bridges::cleanup::verify_recovery(candidate, engine, &receipt, bridge_selection, host_pin)?; + match proof { + SelectedCleanupProof::Absence(witness) => bridges::cleanup::verify_recovery_absence( + candidate, + engine, + &receipt, + bridge_selection, + witness, + )?, + SelectedCleanupProof::Existing(host_pin) => bridges::cleanup::verify_recovery( + candidate, + engine, + &receipt, + bridge_selection, + host_pin, + )?, + } environment::verify_cleanup(candidate, engine, &receipt, &root, environment)?; probes::verify_cleanup(engine, &receipt)?; startup::verify_cleanup(engine, &receipt)?; diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index d43122f1e..9b17ba11b 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -12,6 +12,8 @@ pub use normalized::run_normalized_with_host_dependencies_until; pub use normalized::{ NormalizedInputIdentity, NormalizedRunOptions, compile_normalized_inputs, run_normalized, }; +#[cfg(target_os = "macos")] +mod absent_publication_cleanup; mod admission; mod cache_provenance; mod dependency_hosts; @@ -20,6 +22,10 @@ mod dependency_slots; #[cfg(target_os = "macos")] mod host_pin_recovery; #[cfg(target_os = "macos")] +pub use absent_publication_cleanup::{ + inspect as inspect_absent_publication_cleanup, recover as recover_absent_publication_cleanup, +}; +#[cfg(target_os = "macos")] pub use dependency_slots::{ inspect as dependency_reservations, recover_orphan as recover_dependency_reservation, }; @@ -1567,11 +1573,28 @@ pub fn cleanup( // The caller retains the VM mutation lease through this entire effect and any // additional relay confirmation. Never reconnect from inside this function. fn cleanup_owned( + candidate: &Candidate, + engine: &Engine<'_>, + receipt: Receipt, + root: &std::path::Path, + remove_data: bool, +) -> Result { + cleanup_owned_fenced(candidate, engine, receipt, root, remove_data, false, || { + Ok(()) + }) +} + +/// An explicit recovery can recheck its independent host proof immediately +/// before each destructive graph effect. Ordinary callers use the same engine +/// lease and state machine without an additional recovery witness. +fn cleanup_owned_fenced( candidate: &Candidate, engine: &Engine<'_>, mut receipt: Receipt, root: &std::path::Path, remove_data: bool, + early_intent: bool, + fence: impl Fn() -> Result<(), CandidateError>, ) -> Result { if root.join("state.pending").exists() || root.join("state.pending").is_symlink() { return Err(error( @@ -1579,11 +1602,35 @@ fn cleanup_owned( "Pending graph journal is retained; cleanup is blocked until journal reconciliation.", )); } + fence()?; + // Absence recovery must commit the graph cleanup intent before changing a + // selected relay/bridge inventory. A crash can then resume from an exact + // durable intent even when an early external effect already completed. + let environment_slots = if early_intent { + let slots = environment::cleanup_slots(candidate, engine, &receipt)?; + receipt.phase = "cleanup-intent".into(); + state::write(&root.join("state.json"), &receipt)?; + fence()?; + slots + } else { + Vec::new() + }; startup::cleanup_guest(engine, &receipt, false)?; + fence()?; bridges::release_run(candidate, engine, &receipt)?; - let environment_slots = environment::cleanup_slots(candidate, engine, &receipt)?; - receipt.phase = "cleanup-intent".into(); - state::write(&root.join("state.json"), &receipt)?; + #[cfg(test)] + if early_intent { + fault_pause(root, &receipt.run, "absent-after-bridge-release")?; + } + let environment_slots = if early_intent { + environment_slots + } else { + let slots = environment::cleanup_slots(candidate, engine, &receipt)?; + receipt.phase = "cleanup-intent".into(); + state::write(&root.join("state.json"), &receipt)?; + slots + }; + fence()?; shutdown::stop_owned(engine, &receipt, root)?; for kind in [Kind::Container, Kind::Network, Kind::Volume] { if kind == Kind::Volume && !remove_data { @@ -1605,12 +1652,14 @@ fn cleanup_owned( state::write(&root.join("state.json"), &receipt)?; continue; } + fence()?; if let Some(value) = inspect_resource(engine, &receipt, &resource)? { let target = if kind == Kind::Volume { resource.name.as_str() } else { value["Id"].as_str().expect("verified id") }; + fence()?; engine.request( Method::DELETE, &format!( @@ -1639,12 +1688,15 @@ fn cleanup_owned( fault_pause(root, &receipt.run, "cleanup-after-remove")?; } } + fence()?; startup::cleanup_guest(engine, &receipt, true)?; startup::verify_cleanup(engine, &receipt)?; probes::cleanup(engine, &mut receipt)?; for slot in environment_slots { + fence()?; super::environment_recovery::retire(candidate, engine.guest(), &slot, None)?; } + fence()?; receipt.phase = if remove_data { "removed" } else { @@ -1652,6 +1704,7 @@ fn cleanup_owned( } .into(); state::write(&root.join("state.json"), &receipt)?; + fence()?; Ok(receipt) } diff --git a/packages/runtime-core/tests/absent_recovery_cli_contract.rs b/packages/runtime-core/tests/absent_recovery_cli_contract.rs new file mode 100644 index 000000000..3ea54549d --- /dev/null +++ b/packages/runtime-core/tests/absent_recovery_cli_contract.rs @@ -0,0 +1,107 @@ +//! Recovery acknowledgements must reject at the CLI boundary before any runtime admission. +use serde_json::Value; +use std::{ + fs, + path::{Path, PathBuf}, + process::Command, +}; + +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + let root = std::env::temp_dir().join(format!( + "hack-absence-cli-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&root).unwrap(); + Self(root.canonicalize().unwrap()) + } +} +impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_dir_all(&self.0).unwrap(); + } +} + +#[test] +fn absent_recovery_requires_one_exact_selection_and_both_acknowledgements() { + let fixture = Fixture::new(); + // The development executable intentionally resolves its compiled checkout + // before command dispatch. Supply that valid identity so these assertions + // exercise recovery parsing instead of the unrelated checkout gate. + let checkout = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .unwrap(); + let original = fixture.0.join("original-not-created.json"); + let inspection = fixture.0.join("inspection-not-created.json"); + let complete: Vec = [ + "graph", + "recover-absent-publication-cleanup", + "--run-id", + &"a".repeat(32), + "--original-owner-file", + original.to_str().unwrap(), + "--host-inspection-file", + inspection.to_str().unwrap(), + "--expect-selection", + &"b".repeat(64), + "--retain-data", + "--accept-unpinned-post-reboot", + "--json", + ] + .into_iter() + .map(str::to_owned) + .collect(); + let mut cases = Vec::new(); + for flag in ["--retain-data", "--accept-unpinned-post-reboot"] { + let mut missing = complete.clone(); + missing.retain(|argument| argument != flag); + cases.push(missing); + let mut duplicated = complete.clone(); + duplicated.push(flag.into()); + cases.push(duplicated); + } + for flag in [ + "--run-id", + "--original-owner-file", + "--host-inspection-file", + "--expect-selection", + ] { + let mut missing = complete.clone(); + let position = missing + .iter() + .position(|argument| argument == flag) + .unwrap(); + missing.drain(position..position + 2); + cases.push(missing); + } + for extra in ["--remove-data", "--json", "--force"] { + let mut invalid = complete.clone(); + invalid.push(extra.into()); + cases.push(invalid); + } + for arguments in cases { + let output = Command::new(env!("CARGO_BIN_EXE_hack-runtime-candidate")) + .arg("--candidate-root") + .arg(&checkout) + .args(&arguments) + .env_clear() + .env("PATH", "/nonexistent") + .env("HOME", fixture.0.join("home-not-created")) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2), "{arguments:?}"); + assert!( + output.stdout.is_empty(), + "refusal must not emit a success result" + ); + let failure: Value = serde_json::from_slice(&output.stderr).unwrap(); + assert_eq!(failure["code"], "graph_arguments", "{arguments:?}"); + assert_eq!(fs::read_dir(&fixture.0).unwrap().count(), 0); + } +} From 18fc123b1aff7c32569359ede7351f806d70f7f4 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 03:40:44 -0400 Subject: [PATCH 07/25] fix(runtime): release recovery VM locks with inherited descriptors --- .../runtime-core/src/provider/lifecycle.rs | 24 ++++++++-- .../provider/lifecycle/short_home/tests.rs | 48 +++++++++++++++++++ 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/packages/runtime-core/src/provider/lifecycle.rs b/packages/runtime-core/src/provider/lifecycle.rs index 36decce95..11eba0fdd 100644 --- a/packages/runtime-core/src/provider/lifecycle.rs +++ b/packages/runtime-core/src/provider/lifecycle.rs @@ -1818,8 +1818,25 @@ fn finish_absent( finish_absent_locked(candidate, owner, value, record_disks, &vm_lock) } -/// Keep this descriptor through alias restoration and the stopped receipt. -fn lock_absent_disks(candidate: &Candidate, owner: &Owner) -> Result { +/// Closing alone can leave a flock held by a fork/dup copy until that copy closes. +/// Explicitly unlock this open-file description when the recovery scope ends. +struct VmLock(File); +impl std::ops::Deref for VmLock { + type Target = File; + fn deref(&self) -> &File { + &self.0 + } +} +impl Drop for VmLock { + fn drop(&mut self) { + use std::os::fd::AsRawFd; + // SAFETY: the owned descriptor remains live through this Drop call. + unsafe { libc::flock(self.0.as_raw_fd(), libc::LOCK_UN) }; + } +} + +/// Keep this guard through alias restoration and the stopped receipt. +fn lock_absent_disks(candidate: &Candidate, owner: &Owner) -> Result { let directory = owner.real_data_dir(candidate)?; let vm_lock = OpenOptions::new() .read(true) @@ -1833,13 +1850,14 @@ fn lock_absent_disks(candidate: &Candidate, owner: &Owner) -> Result Date: Wed, 30 Sep 2026 03:42:06 -0400 Subject: [PATCH 08/25] test(runtime): preserve private recovery inspection selection --- .../absent_publication_recovery.rs | 79 ++++++++++++++++++- 1 file changed, 75 insertions(+), 4 deletions(-) diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs b/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs index aa290d1d8..5fe2c3c39 100644 --- a/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs +++ b/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs @@ -4,7 +4,10 @@ use super::dependency_rebind::{checked_cli, exec, snapshot}; use super::*; use crate::provider::{lifecycle, state::Owner}; -use std::os::unix::fs::MetadataExt; +use std::{ + io::Write, + os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt}, +}; fn ready(owner: &mut Process, run: &str, deadline: Instant) { loop { @@ -306,6 +309,8 @@ fn explicit_absent_publication_cleanup_retains_selected_volume_and_sibling() { .unwrap(); let graph_root = graph::directory(&candidate, &runs[0]).unwrap(); let raw_graph = fs::read(graph_root.join("state.json")).unwrap(); + let provider_owner_path = candidate.state_root.join("run/smolvm/owner.json"); + let raw_provider_owner = fs::read(&provider_owner_path).unwrap(); let publisher_root = transport::root(&candidate, &runs[0]).unwrap(); let control_root = original .relay_startup @@ -318,9 +323,61 @@ fn explicit_absent_publication_cleanup_retains_selected_volume_and_sibling() { fs::remove_dir_all(&control_root).unwrap(); let old = old_path.to_str().unwrap(); - let prior = inspection_path.to_str().unwrap(); + let original_inspection = inspection_path.to_str().unwrap(); + let original_selection = checked_cli( + "inspect-private-original-inspection", + &binary, + &candidate, + &inspect_args(&runs[0], old, original_inspection), + deadline, + ); + let original_selection_hash = original_selection["selection_sha256"].as_str().unwrap(); + assert_eq!(original_selection_hash.len(), 64); + let inspection_bytes = fs::read(&inspection_path).unwrap(); + fs::set_permissions(&inspection_path, fs::Permissions::from_mode(0o644)).unwrap(); + assert_eq!( + fs::symlink_metadata(&inspection_path).unwrap().mode() & 0o7777, + 0o644 + ); + refused_cli( + &binary, + &candidate, + &inspect_args(&runs[0], old, original_inspection), + "graph_absent_publication_recovery", + deadline, + ); + assert!( + !publisher_root.exists(), + "read-only refusal did not reserve a publisher" + ); + assert_eq!(fs::read(&provider_owner_path).unwrap(), raw_provider_owner); + assert_eq!(fs::read(graph_root.join("state.json")).unwrap(), raw_graph); + + // A new private path retains the exact inspected bytes, while its path is + // deliberately part of the fresh selection hash. The historical 0644 + // receipt remains untouched after the copy. + let private_inspection = private.0.join("private-inspection-copy.json"); + let mut copy = fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&private_inspection) + .unwrap(); + copy.write_all(&inspection_bytes).unwrap(); + copy.sync_all().unwrap(); + fs::File::open(&private.0).unwrap().sync_all().unwrap(); + assert_eq!(fs::read(&private_inspection).unwrap(), inspection_bytes); + assert_eq!( + fs::symlink_metadata(&private_inspection).unwrap().mode() & 0o7777, + 0o600 + ); + assert_eq!( + fs::symlink_metadata(&inspection_path).unwrap().mode() & 0o7777, + 0o644 + ); + let prior = private_inspection.to_str().unwrap(); let initial = checked_cli( - "inspect-absent-publications", + "inspect-exact-private-inspection-copy", &binary, &candidate, &inspect_args(&runs[0], old, prior), @@ -328,7 +385,21 @@ fn explicit_absent_publication_cleanup_retains_selected_volume_and_sibling() { ); let selection = initial["selection_sha256"].as_str().unwrap(); assert_eq!(selection.len(), 64); + assert_ne!( + selection, original_selection_hash, + "selection binds the new canonical path" + ); + assert_eq!(fs::read(&provider_owner_path).unwrap(), raw_provider_owner); + assert_eq!(fs::read(graph_root.join("state.json")).unwrap(), raw_graph); let intent_path = graph_root.join("absent-publication-cleanup.json"); + assert!(!intent_path.exists()); + refused_cli( + &binary, + &candidate, + &recover_args(&runs[0], old, prior, original_selection_hash), + "graph_absent_publication_recovery", + deadline, + ); refused_cli( &binary, &candidate, @@ -360,7 +431,7 @@ fn explicit_absent_publication_cleanup_retains_selected_volume_and_sibling() { assert!(!intent_path.exists()); } { - let _missing = Moved::new(inspection_path.clone()); + let _missing = Moved::new(private_inspection.clone()); refused_cli( &binary, &candidate, From 8e76aa5bf2f676c7c901bec055360aac5b39a4af Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 03:47:09 -0400 Subject: [PATCH 09/25] test(runtime): isolate concurrent recovery fixture roots --- .../provider/lifecycle/short_home/tests.rs | 79 ++++++++++++++++--- 1 file changed, 67 insertions(+), 12 deletions(-) diff --git a/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs b/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs index d914962ea..2654463e9 100644 --- a/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs +++ b/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs @@ -4,12 +4,71 @@ use std::{ fs, os::{ fd::AsRawFd, - unix::{fs::MetadataExt, net::UnixListener}, + unix::{ + fs::{DirBuilderExt, MetadataExt}, + net::UnixListener, + }, }, path::PathBuf, process::Command, + sync::atomic::{AtomicU64, Ordering}, }; +static NEXT_POOL: AtomicU64 = AtomicU64::new(0); + +fn pool_directory(timestamp: u128) -> PathBuf { + fs::canonicalize(std::env::temp_dir()) + .unwrap() + .join(format!( + "hack-home-recovery-{}-{timestamp}-{}", + std::process::id(), + NEXT_POOL.fetch_add(1, Ordering::Relaxed) + )) +} + +fn create_pool_directory(path: &std::path::Path) -> std::io::Result<()> { + fs::DirBuilder::new().mode(0o700).create(path) +} + +#[test] +fn parallel_pool_roots_are_unique_at_the_same_timestamp_and_never_adopt() { + let paths: Vec<_> = std::thread::scope(|scope| { + let workers: Vec<_> = (0..16) + .map(|_| { + scope.spawn(|| { + let path = pool_directory(123); + create_pool_directory(&path).unwrap(); + path + }) + }) + .collect(); + workers + .into_iter() + .map(|worker| worker.join().unwrap()) + .collect() + }); + assert_eq!( + paths + .iter() + .collect::>() + .len(), + 16 + ); + for path in paths { + let before = fs::symlink_metadata(&path).unwrap(); + assert_eq!(before.mode() & 0o777, 0o700); + fs::write(path.join("marker"), b"owned fixture").unwrap(); + assert_eq!( + create_pool_directory(&path).unwrap_err().kind(), + std::io::ErrorKind::AlreadyExists + ); + assert_eq!(fs::symlink_metadata(&path).unwrap().ino(), before.ino()); + assert_eq!(fs::read(path.join("marker")).unwrap(), b"owned fixture"); + fs::remove_file(path.join("marker")).unwrap(); + fs::remove_dir(path).unwrap(); + } +} + struct Pool { candidate: Candidate, owner: Owner, @@ -21,17 +80,13 @@ impl Pool { let mut process = identity::observe(child.id() as i32).unwrap(); child.kill().unwrap(); child.wait().unwrap(); - let directory = fs::canonicalize(std::env::temp_dir()) - .unwrap() - .join(format!( - "hack-home-recovery-{}-{}", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - state::private_directory(&directory).unwrap(); + let directory = pool_directory( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(), + ); + create_pool_directory(&directory).unwrap(); let candidate = Candidate::discover(&directory).unwrap(); let operation = state::Lock::acquire(&root(&candidate)).unwrap(); let mut owner = Owner::create( From 318b39c393d2a342c19dc0b79592061c501698d2 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 03:59:05 -0400 Subject: [PATCH 10/25] fix(runtime): restore retained source after host device migration --- packages/runtime-core/README.md | 30 + packages/runtime-core/src/graph_cli.rs | 44 ++ packages/runtime-core/src/main.rs | 2 + .../src/provider/graph/foreground.rs | 12 +- .../provider/graph/foreground/native_test.rs | 1 + .../absent_publication_recovery.rs | 13 +- .../native_test/source_device_rebind.rs | 562 +++++++++++++++ .../src/provider/graph/foreground/tests.rs | 19 + .../provider/graph/foreground/transport.rs | 4 + .../runtime-core/src/provider/graph/mod.rs | 25 +- .../src/provider/graph/restore.rs | 95 ++- .../runtime-core/src/provider/graph/source.rs | 10 +- .../provider/graph/source_device_rebind.rs | 642 ++++++++++++++++++ .../source_device_rebind_cli_contract.rs | 62 ++ 14 files changed, 1503 insertions(+), 18 deletions(-) create mode 100644 packages/runtime-core/src/provider/graph/foreground/native_test/source_device_rebind.rs create mode 100644 packages/runtime-core/src/provider/graph/source_device_rebind.rs create mode 100644 packages/runtime-core/tests/source_device_rebind_cli_contract.rs diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index dd830284f..cac696a41 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -803,3 +803,33 @@ before fresh dependency ownership is admitted. This does not enable file-based `graph restart`/`restore` or normalized live-owner redelivery, and does not replay initializer cache-release effects. Native same-run restoration and cross-boot volume continuity require separate runtime qualification. + +An explicitly recovered post-reboot graph may retain a stopped shared-source +receipt whose `source.shared.device` names the prior host filesystem device. +After **completed** selected absent-publication cleanup and separate publisher +retirement, inspect that same run with `graph inspect-source-device-rebind +--run-id RUN --json`. Review the returned original stopped-receipt hash and +qualification, then commit only that selection with `graph +recover-source-device-rebind --run-id RUN --expect-selection SHA +--accept-legacy-device-rebind --json`. This private candidate command changes no +stopped receipt, cleanup proof, retirement proof or named volume. It writes one +immutable witness for the current Owner share, requiring the same canonical +project path, inode, guest path and unfiltered access, with only the host device +number changed. The current guest's writable virtiofs mount and selected volume +identities are checked independently. The explicit acceptance records that +legacy receipts cannot establish original physical volume continuity across the +host reboot. + +`restore-selection` then binds the witness's raw hash to the selected generation. +The source checks use only an in-memory device projection; original retention and +restore history consume the unchanged stopped receipt. The new receipt receives +the projected source after history retention, before the first new-attempt write. +The witness file remains pinned and is rechecked before that write. Once the +first new generation replaces the stopped receipt, the witness is audit history +and grants no authority to later ordinary down/up cycles. An incomplete +`source-device-rebind.pending` file, including an empty or partial write, is +preserved and refuses both explicit recovery and a new retired-publisher bind; +there is no automatic prefix-based adoption or deletion. The ignored native +synthetic-device fixture checks this same-run transition and later retained +marker reads, but only an actual host-reboot application run can qualify the +physical continuity claim. diff --git a/packages/runtime-core/src/graph_cli.rs b/packages/runtime-core/src/graph_cli.rs index d962c917f..9bf84625d 100644 --- a/packages/runtime-core/src/graph_cli.rs +++ b/packages/runtime-core/src/graph_cli.rs @@ -150,6 +150,50 @@ pub fn command(candidate: &Candidate, args: &[&str]) -> Result run, + _ => return Err(invalid()), + }; + #[cfg(target_os = "macos")] + { + return graph::inspect_source_device_rebind(candidate, run); + } + #[cfg(not(target_os = "macos"))] + { + let _ = run; + return Err(invalid()); + } + } + if *action == "recover-source-device-rebind" { + let (run, expected) = match *args { + [ + "--run-id", + run, + "--expect-selection", + expected, + "--accept-legacy-device-rebind", + ] + | [ + "--run-id", + run, + "--expect-selection", + expected, + "--accept-legacy-device-rebind", + "--json", + ] => (run, expected), + _ => return Err(invalid()), + }; + #[cfg(target_os = "macos")] + { + return graph::recover_source_device_rebind(candidate, run, expected); + } + #[cfg(not(target_os = "macos"))] + { + let _ = (run, expected); + return Err(invalid()); + } + } if ["run-selection", "run-service"].contains(action) { return one_off::command(candidate, action, args); } diff --git a/packages/runtime-core/src/main.rs b/packages/runtime-core/src/main.rs index 558aff288..b4c53ad58 100644 --- a/packages/runtime-core/src/main.rs +++ b/packages/runtime-core/src/main.rs @@ -31,6 +31,8 @@ Usage: hack-local graph recover-host-pins --run-id <32-hex> --expect-selection <64-hex> --accept-legacy-device-rebind [--json] hack-local graph inspect-absent-publication-cleanup --run-id <32-hex> --original-owner-file --host-inspection-file [--json] hack-local graph recover-absent-publication-cleanup --run-id <32-hex> --original-owner-file --host-inspection-file --expect-selection <64-hex> --retain-data --accept-unpinned-post-reboot [--json] + hack-local graph inspect-source-device-rebind --run-id <32-hex> [--json] + hack-local graph recover-source-device-rebind --run-id <32-hex> --expect-selection <64-hex> --accept-legacy-device-rebind [--json] hack-local graph logs --run-id <32-hex> --service [--tail <1..1000>] [--json] hack-local graph exec --run-id <32-hex> --service [--workdir /path] [--timeout-seconds <1..120>] [--json] -- [args...] hack-local graph dependency-plan --dependencies [--json] diff --git a/packages/runtime-core/src/provider/graph/foreground.rs b/packages/runtime-core/src/provider/graph/foreground.rs index 366769dc6..5e1085959 100644 --- a/packages/runtime-core/src/provider/graph/foreground.rs +++ b/packages/runtime-core/src/provider/graph/foreground.rs @@ -163,10 +163,20 @@ pub fn restore_selection(candidate: &Candidate, run: &str) -> Result(run: &'a str, old: &'a str, prior: &'a str) -> [&'a str; 9] { +pub(super) fn inspect_args<'a>(run: &'a str, old: &'a str, prior: &'a str) -> [&'a str; 9] { [ "graph", "inspect-absent-publication-cleanup", @@ -58,7 +58,12 @@ fn inspect_args<'a>(run: &'a str, old: &'a str, prior: &'a str) -> [&'a str; 9] "--json", ] } -fn recover_args<'a>(run: &'a str, old: &'a str, prior: &'a str, hash: &'a str) -> [&'a str; 13] { +pub(super) fn recover_args<'a>( + run: &'a str, + old: &'a str, + prior: &'a str, + hash: &'a str, +) -> [&'a str; 13] { [ "graph", "recover-absent-publication-cleanup", diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test/source_device_rebind.rs b/packages/runtime-core/src/provider/graph/foreground/native_test/source_device_rebind.rs new file mode 100644 index 000000000..7c4129c54 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/foreground/native_test/source_device_rebind.rs @@ -0,0 +1,562 @@ +//! Owned synthetic-device fixture. Only a physical host reboot can qualify +//! original APFS volume continuity for an application; this fixture checks the +//! selected source transition, original receipt/history and retained marker. +use super::absent_publication_recovery::{inspect_args, ready, recover_args, refused_cli}; +use super::dependency_rebind::{checked_cli, exec, snapshot}; +use super::*; +use crate::provider::{ProjectShareIntent, lifecycle, state::Owner}; +use base64::Engine as _; +use std::os::unix::fs::MetadataExt; + +fn shared_source_marker( + binary: &Path, + candidate: &Candidate, + run: &str, + expected: &[u8], + deadline: Instant, +) { + let mut process = Process::start( + binary, + candidate, + &[ + "graph", + "exec", + "--run-id", + run, + "--service", + "web", + "--timeout-seconds", + "12", + "--json", + "--", + "/bin/sh", + "-ec", + "cat /app/source-marker.txt", + ], + None, + ); + let status = process.wait(deadline); + let result: Value = serde_json::from_slice(&process.out).unwrap_or(Value::Null); + assert!(status.success(), "shared-source exec CLI refused"); + assert_eq!(result["exit_code"], 0, "shared-source process failed"); + assert_eq!(result["truncated"], false); + let stdout = base64::engine::general_purpose::STANDARD + .decode(result["stdout_base64"].as_str().unwrap()) + .unwrap(); + let stderr = base64::engine::general_purpose::STANDARD + .decode(result["stderr_base64"].as_str().unwrap()) + .unwrap(); + assert!(stderr.is_empty()); + assert_eq!(stdout, expected); +} + +struct ReplacedBytes { + path: PathBuf, + original: Vec, + substituted: Vec, +} +impl ReplacedBytes { + fn new(path: &Path, bytes: Vec) -> Self { + let original = fs::read(path).unwrap(); + fs::write(path, &bytes).unwrap(); + Self { + path: path.to_owned(), + original, + substituted: bytes, + } + } +} +impl Drop for ReplacedBytes { + fn drop(&mut self) { + if fs::read(&self.path).ok().as_deref() == Some(self.substituted.as_slice()) { + fs::write(&self.path, &self.original).unwrap(); + } + } +} + +#[test] +#[ignore = "Owned development VM, private synthetic prior-device fault, pinned image and 300s watchdog required"] +fn selected_source_device_rebind_restores_same_run_and_two_ordinary_generations() { + let deadline = Instant::now() + Duration::from_secs(270); + let candidate = + Candidate::discover(Path::new(&std::env::var("HACK_LOCAL_TEST_ROOT").unwrap())).unwrap(); + let binary = PathBuf::from(std::env::var("HACK_LOCAL_TEST_BINARY").unwrap()); + let image = std::env::var("HACK_LOCAL_TEST_IMAGE").unwrap(); + let image_archive = std::env::var("HACK_LOCAL_TEST_IMAGE_ARCHIVE").unwrap(); + let image_sha256 = std::env::var("HACK_LOCAL_TEST_IMAGE_SHA256").unwrap(); + let fixture = graph::tests::Fixture::new(); + let project = fixture.0.join("project"); + fs::create_dir(&project).unwrap(); + state::write( + &project.join("compose.yaml"), + &json!({ + "services":{"web":{"image":image,"read_only":true,"network_mode":"none", + "init":true,"user":"0:0","entrypoint":["/bin/sleep","300"],"command":[], + "volumes":["data:/data",".:/app:ro"], + "healthcheck":{"test":["CMD","/bin/hack-graph-startup-app","complete"], + "interval":"200ms","timeout":"2s","retries":10,"start_period":"500ms"}}}, + "volumes":{"data":{}} + }), + ) + .unwrap(); + let marker_path = project.join("source-marker.txt"); + let initial_marker = b"source-initial\n"; + fs::write(&marker_path, initial_marker).unwrap(); + // This fixture starts an otherwise uninitialized owned pool with one exact + // approved source root; the external harness supplies the candidate binary. + assert_eq!( + lifecycle::status(&candidate).unwrap().phase, + "uninitialized" + ); + let share = ProjectShareIntent::approve(&project, true).unwrap(); + lifecycle::up_with_project_share( + &candidate, + crate::provider::Profile::Development, + None, + None, + None, + Some(share.clone()), + ) + .unwrap(); + checked_cli( + "load-pinned-image", + &binary, + &candidate, + &[ + "runtime", + "load-image", + "--archive", + &image_archive, + "--sha256", + &image_sha256, + "--image-id", + &image, + ], + deadline, + ); + let run = graph::probes::token().unwrap(); + let review = project::plan( + &candidate, + project::PlanOptions { + branch: None, + project: &project, + compose_file: Path::new("compose.yaml"), + profiles: &[], + }, + ) + .unwrap(); + let normalized = project.join("compose.yaml"); + let publication = checked_cli( + "publish-source", + &binary, + &candidate, + &[ + "project", + "publish-source", + "--project", + project.to_str().unwrap(), + "--file", + "compose.yaml", + "--expect-plan", + &review.plan_id, + "--normalized-file", + normalized.to_str().unwrap(), + "--expect-original", + &review.plan.compose_sha256, + "--expect-namespace", + &review.plan.namespace, + "--json", + ], + deadline, + ); + let revision = publication["revision"].as_str().unwrap(); + let dependencies_path = fixture.0.join("dependencies.json"); + state::write( + &dependencies_path, + &json!({"version":1,"plan":review.plan_id, + "artifact":"/tmp/unused-control-only-artifact","artifact_sha256":"a".repeat(64), + "dependencies":[]}), + ) + .unwrap(); + let dependency = checked_cli( + "dependency-plan", + &binary, + &candidate, + &[ + "graph", + "dependency-plan", + "--dependencies", + dependencies_path.to_str().unwrap(), + "--json", + ], + deadline, + ); + let start = |generation: Option<&str>| { + let mut args = vec![ + "graph", + if generation.is_some() { + "serve-restore" + } else { + "serve" + }, + "--project", + project.to_str().unwrap(), + "--file", + "compose.yaml", + "--expect-plan", + &review.plan_id, + "--run-id", + &run, + "--ready", + "web=healthy", + "--timeout-seconds", + "90", + "--dependencies", + dependencies_path.to_str().unwrap(), + "--expect-dependencies", + dependency["dependency_plan_id"].as_str().unwrap(), + "--source-revision", + revision, + "--shared-source", + "--normalized-file", + normalized.to_str().unwrap(), + "--expect-original", + &review.plan.compose_sha256, + "--expect-namespace", + &review.plan.namespace, + "--json", + ]; + if let Some(generation) = generation { + args.extend(["--expect-generation", generation]); + } + Process::start(&binary, &candidate, &args, None) + }; + let mut owner = start(None); + let mut cleanup = Cleanup { + binary: &binary, + candidate: &candidate, + run: &run, + done: false, + }; + ready(&mut owner, &run, deadline); + shared_source_marker(&binary, &candidate, &run, initial_marker, deadline); + assert_eq!( + exec(&binary, &candidate, &run, "web", "write-data", deadline)["exit_code"], + 0 + ); + let ready_receipt = snapshot(&candidate, &run, deadline).receipt; + let original_volume = ready_receipt.resources["volume:data"].name.clone(); + let original_container = ready_receipt.resources["container:web"].id.clone(); + let old_owner = Owner::load(&candidate).unwrap(); + owner.child.kill().unwrap(); + owner.wait(deadline); + assert_eq!(lifecycle::down(&candidate).unwrap().phase, "stopped"); + lifecycle::up(&candidate).unwrap(); + let current = Owner::load(&candidate).unwrap(); + assert_eq!(current.project_share.as_ref(), Some(&share)); + let old_device = share.device.checked_add(1).unwrap(); + let boot = lifecycle::host_filesystem::host_boot_micros().unwrap(); + let mut synthetic_owner = old_owner.clone(); + synthetic_owner.storage.as_mut().unwrap().device = old_device; + synthetic_owner.overlay.as_mut().unwrap().device = old_device; + synthetic_owner.project_share.as_mut().unwrap().device = old_device; + synthetic_owner.process.as_mut().unwrap().pid = i32::MAX; + synthetic_owner.process.as_mut().unwrap().start_micros = boot - 1; + let old_owner_path = fixture.0.join("prior-owner.json"); + state::write(&old_owner_path, &synthetic_owner).unwrap(); + let old_bytes = fs::read(&old_owner_path).unwrap(); + let pool = fs::symlink_metadata(candidate.state_root.join("run/smolvm")).unwrap(); + let inspection_path = fixture.0.join("prior-inspection.json"); + state::write( + &inspection_path, + &serde_json::from_slice::(&graph::absent_publication_cleanup::fixture_inspection( + &old_bytes, + ¤t, + old_device, + boot, + pool.ino(), + )) + .unwrap(), + ) + .unwrap(); + let graph_root = graph::directory(&candidate, &run).unwrap(); + // Test-only host-device fault: a real reboot changes st_dev without + // rewriting the original graph. This isolated synthetic receipt is fixed + // before selection and remains byte-for-byte immutable thereafter. + let mut synthetic_ready = ready_receipt.clone(); + synthetic_ready + .source + .as_mut() + .unwrap() + .shared + .as_mut() + .unwrap() + .device = old_device; + state::write(&graph_root.join("state.json"), &synthetic_ready).unwrap(); + let original_ready_bytes = fs::read(graph_root.join("state.json")).unwrap(); + let publisher_root = transport::root(&candidate, &run).unwrap(); + fs::remove_dir_all(&publisher_root).unwrap(); + fs::remove_dir_all( + synthetic_ready + .relay_startup + .as_ref() + .unwrap() + .control_root + .clone(), + ) + .unwrap(); + let prior = inspection_path.to_str().unwrap(); + let old = old_owner_path.to_str().unwrap(); + let absent_selection = checked_cli( + "select-absence", + &binary, + &candidate, + &inspect_args(&run, old, prior), + deadline, + ); + let absent_hash = absent_selection["selection_sha256"].as_str().unwrap(); + let stopped = checked_cli( + "recover-absence", + &binary, + &candidate, + &recover_args(&run, old, prior, absent_hash), + deadline, + ); + assert_eq!(stopped["phase"], "stopped-data-retained"); + let original_stopped_bytes = fs::read(graph_root.join("state.json")).unwrap(); + let original_stopped: graph::Receipt = serde_json::from_slice(&original_stopped_bytes).unwrap(); + assert_eq!( + original_stopped + .source + .as_ref() + .unwrap() + .shared + .as_ref() + .unwrap() + .device, + old_device + ); + let original_history_hash = format!("{:x}", Sha256::digest(&original_stopped_bytes)); + let witness_path = graph_root.join("source-device-rebind.json"); + let inspect = checked_cli( + "select-source", + &binary, + &candidate, + &[ + "graph", + "inspect-source-device-rebind", + "--run-id", + &run, + "--json", + ], + deadline, + ); + let witness_hash = inspect["selection_sha256"].as_str().unwrap(); + refused_cli( + &binary, + &candidate, + &[ + "graph", + "recover-source-device-rebind", + "--run-id", + &run, + "--expect-selection", + &"0".repeat(64), + "--accept-legacy-device-rebind", + "--json", + ], + "graph_source_device_rebind", + deadline, + ); + assert!(!witness_path.exists()); + let pending = witness_path.with_extension("pending"); + fs::write(&pending, b"foreign incomplete witness").unwrap(); + refused_cli( + &binary, + &candidate, + &[ + "graph", + "recover-source-device-rebind", + "--run-id", + &run, + "--expect-selection", + witness_hash, + "--accept-legacy-device-rebind", + "--json", + ], + "graph_source_device_rebind", + deadline, + ); + assert_eq!(fs::read(&pending).unwrap(), b"foreign incomplete witness"); + fs::remove_file(&pending).unwrap(); // Only this fixture's known pending file. + checked_cli( + "commit-source", + &binary, + &candidate, + &[ + "graph", + "recover-source-device-rebind", + "--run-id", + &run, + "--expect-selection", + witness_hash, + "--accept-legacy-device-rebind", + "--json", + ], + deadline, + ); + assert_eq!( + fs::read(graph_root.join("state.json")).unwrap(), + original_stopped_bytes + ); + let committed_bytes = fs::read(&witness_path).unwrap(); + { + let mut changed = committed_bytes.clone(); + changed.push(b' '); + let _changed = ReplacedBytes::new(&witness_path, changed); + refused_cli( + &binary, + &candidate, + &["graph", "restore-selection", "--run-id", &run, "--json"], + "graph_source_device_rebind", + deadline, + ); + } + { + let _changed = ReplacedBytes::new( + &witness_path, + graph::source_device_rebind::fixture_wrong_volume_projection(&committed_bytes), + ); + refused_cli( + &binary, + &candidate, + &["graph", "restore-selection", "--run-id", &run, "--json"], + "graph_source_device_rebind", + deadline, + ); + } + assert_eq!(fs::read(&witness_path).unwrap(), committed_bytes); + let selection = checked_cli( + "restore-selection-source", + &binary, + &candidate, + &["graph", "restore-selection", "--run-id", &run, "--json"], + deadline, + ); + let selected_generation = selection["generation"].as_str().unwrap(); + fs::write(&pending, b"appeared after selection").unwrap(); + let mut blocked = start(Some(selected_generation)); + assert_eq!(blocked.wait(deadline).code(), Some(2)); + assert!(blocked.out.is_empty()); + let refusal: Value = serde_json::from_slice(&blocked.err).unwrap(); + assert_eq!(refusal["code"], "graph_source_device_rebind"); + assert_eq!(fs::read(&pending).unwrap(), b"appeared after selection"); + assert!(!publisher_root.join("control.sock").exists()); + assert!(!publisher_root.join("owner.json").exists()); + fs::remove_file(&pending).unwrap(); // The fixture owns this exact pending file. + owner = start(Some(selected_generation)); + ready(&mut owner, &run, deadline); + shared_source_marker(&binary, &candidate, &run, initial_marker, deadline); + assert_eq!( + exec(&binary, &candidate, &run, "web", "read-data", deadline)["exit_code"], + 0 + ); + let restored = snapshot(&candidate, &run, deadline).receipt; + assert_eq!(restored.resources["volume:data"].name, original_volume); + assert_ne!(restored.resources["container:web"].id, original_container); + assert_eq!( + restored + .source + .as_ref() + .unwrap() + .shared + .as_ref() + .unwrap() + .device, + share.device + ); + assert!(fs::read(graph_root.join("restore-history.json")).is_ok()); + assert_eq!(fs::read(&witness_path).unwrap(), committed_bytes); + assert_eq!( + fs::read(graph_root.join("state.json")).unwrap(), + serde_json::to_vec_pretty(&restored).unwrap() + ); + assert!( + graph::restore_history::completed_for_recovery( + &graph_root, + &restored, + &original_history_hash + ) + .unwrap() + .is_some() + ); + assert_ne!( + fs::read(graph_root.join("state.json")).unwrap(), + original_ready_bytes + ); + fs::write(&marker_path, b"source-host-edit\n").unwrap(); + shared_source_marker(&binary, &candidate, &run, b"source-host-edit\n", deadline); + fs::write(&marker_path, initial_marker).unwrap(); + shared_source_marker(&binary, &candidate, &run, initial_marker, deadline); + + // The selected witness is now historical. Each later cleanup/restore takes + // an ordinary generation and reads the same named retained data. + for generation in 0..2 { + let cleaned = checked_cli( + "ordinary-cleanup", + &binary, + &candidate, + &["graph", "cleanup", "--run-id", &run, "--json"], + deadline, + ); + assert_eq!(cleaned["phase"], "stopped-data-retained"); + assert!(owner.wait(deadline).success()); + let next = checked_cli( + "ordinary-selection", + &binary, + &candidate, + &["graph", "restore-selection", "--run-id", &run, "--json"], + deadline, + ); + owner = start(Some(next["generation"].as_str().unwrap())); + ready(&mut owner, &run, deadline); + shared_source_marker(&binary, &candidate, &run, initial_marker, deadline); + assert_eq!( + exec(&binary, &candidate, &run, "web", "read-data", deadline)["exit_code"], + 0 + ); + let observed = snapshot(&candidate, &run, deadline).receipt; + assert_eq!( + observed.resources["volume:data"].name, original_volume, + "ordinary generation {generation}" + ); + assert_eq!( + observed + .source + .as_ref() + .unwrap() + .shared + .as_ref() + .unwrap() + .device, + share.device + ); + } + let removed = checked_cli( + "remove-selected-data", + &binary, + &candidate, + &[ + "graph", + "cleanup", + "--run-id", + &run, + "--remove-data", + "--json", + ], + deadline, + ); + assert_eq!(removed["phase"], "removed"); + assert!(owner.wait(deadline).success()); + cleanup.done = true; +} diff --git a/packages/runtime-core/src/provider/graph/foreground/tests.rs b/packages/runtime-core/src/provider/graph/foreground/tests.rs index 3ba36a097..2c9e033b3 100644 --- a/packages/runtime-core/src/provider/graph/foreground/tests.rs +++ b/packages/runtime-core/src/provider/graph/foreground/tests.rs @@ -47,6 +47,25 @@ fn fresh_restore_publication_requires_prior_retirement_and_exclusive_lock() { drop(second); fs::remove_dir_all(path).unwrap(); } +#[cfg(target_os = "macos")] +#[test] +fn retired_publication_refuses_incomplete_source_device_witness_before_binding() { + let (_fixture, candidate, run) = fixture(); + let publisher = root(&candidate, &run); + let mut first = Publication::bind(&candidate, &run).unwrap(); + first.finish().unwrap(); + drop(first); + let graph_root = super::super::directory(&candidate, &run).unwrap(); + crate::provider::state::private_directory(&graph_root).unwrap(); + let pending = graph_root.join("source-device-rebind.pending"); + fs::write(&pending, b"foreign incomplete witness").unwrap(); + + assert!(Publication::bind_retired(&candidate, &run).is_err()); + assert!(!publisher.join("control.sock").exists()); + assert!(!publisher.join("owner.json").exists()); + assert_eq!(fs::read(&pending).unwrap(), b"foreign incomplete witness"); + fs::remove_dir_all(publisher).unwrap(); +} #[test] fn publication_serializes_and_pins_exact_server_socket() { let (_fixture, candidate, run) = fixture(); diff --git a/packages/runtime-core/src/provider/graph/foreground/transport.rs b/packages/runtime-core/src/provider/graph/foreground/transport.rs index 975cb8540..d92072cf9 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport.rs @@ -783,6 +783,10 @@ impl Publication { }; // The foreground lock serializes a concurrent absence-intent writer. super::super::absent_publication_cleanup::publication_allowed(candidate, run, retired)?; + #[cfg(target_os = "macos")] + if retired { + super::super::source_device_rebind::require_no_pending(candidate, run)?; + } for name in ["control.sock", "owner.json"] { match fs::symlink_metadata(root.join(name)) { Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index 9b17ba11b..1001b3330 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -26,11 +26,17 @@ pub use absent_publication_cleanup::{ inspect as inspect_absent_publication_cleanup, recover as recover_absent_publication_cleanup, }; #[cfg(target_os = "macos")] +mod source_device_rebind; +#[cfg(target_os = "macos")] pub use dependency_slots::{ inspect as dependency_reservations, recover_orphan as recover_dependency_reservation, }; #[cfg(target_os = "macos")] pub use host_pin_recovery::{inspect as inspect_host_pin_recovery, recover as recover_host_pins}; +#[cfg(target_os = "macos")] +pub use source_device_rebind::{ + inspect as inspect_source_device_rebind, recover as recover_source_device_rebind, +}; mod initializer_cache; mod volume_subpaths; pub use dependency_hosts::dependency_address; @@ -1446,6 +1452,14 @@ pub fn source_compatibility( } let engine = Engine::connect_cleanup(candidate)?; let (receipt, root) = load(candidate, &engine, run)?; + #[cfg(target_os = "macos")] + let source_rebind = source_device_rebind::select(&engine, &receipt, &root)?; + #[cfg(target_os = "macos")] + let source_receipt = source_rebind + .as_ref() + .map_or(&receipt, |selected| selected.source_receipt()); + #[cfg(not(target_os = "macos"))] + let source_receipt = &receipt; if !hex(plan_id, 64) || project::identity(plan)? != plan_id || !normalized.matches_plan(plan) @@ -1463,7 +1477,7 @@ pub fn source_compatibility( .services .values() .any(|service| service.active && service.dependency_cache.is_some()); - let revision = receipt + let revision = source_receipt .source .as_ref() .map(|binding| binding.revision.as_str()); @@ -1476,8 +1490,13 @@ pub fn source_compatibility( } if !unchanged_normalized_review(&receipt, plan_id, normalized) { let revision = revision.ok_or_else(refused)?; - if hostname_change::prepare(&engine, plan, &receipt, normalized)?.is_none() { - source::prepare_shared_changed(&engine, plan, &receipt, cached.then_some(revision))?; + if hostname_change::prepare(&engine, plan, source_receipt, normalized)?.is_none() { + source::prepare_shared_changed( + &engine, + plan, + source_receipt, + cached.then_some(revision), + )?; } } Ok(json!({ diff --git a/packages/runtime-core/src/provider/graph/restore.rs b/packages/runtime-core/src/provider/graph/restore.rs index 4b8c7532a..72a2c7c54 100644 --- a/packages/runtime-core/src/provider/graph/restore.rs +++ b/packages/runtime-core/src/provider/graph/restore.rs @@ -149,11 +149,10 @@ fn verify_fresh_change( /// Binds the stopped selection to current boot and exact observed retained data. /// Existing receipts identify managed volumes by name/labels; this additionally /// fences replacement between explicit selection and restoration. -pub(super) fn restore_generation( +pub(super) fn observed_volumes( engine: &Engine<'_>, receipt: &Receipt, -) -> Result { - use sha2::{Digest, Sha256}; +) -> Result, CandidateError> { let mut volumes = BTreeMap::new(); for (key, resource) in &receipt.resources { if resource.kind != Kind::Volume { @@ -172,12 +171,44 @@ pub(super) fn restore_generation( "Retained volume changed during selection.", )); } - volumes.insert(key, (resource.name.as_str(), created.to_owned(), directory)); + volumes.insert( + key.clone(), + (resource.name.clone(), created.to_owned(), directory), + ); } - let bytes = serde_json::to_vec(&(receipt, engine.guest().boot_id(), volumes)) + Ok(volumes) +} + +pub(super) fn restore_generation( + engine: &Engine<'_>, + receipt: &Receipt, +) -> Result { + use sha2::{Digest, Sha256}; + let volumes = observed_volumes(engine, receipt)?; + let bytes = serde_json::to_vec(&(receipt, engine.guest().boot_id(), &volumes)) .map_err(|_| error("graph_receipt", "Restore selection unavailable."))?; Ok(format!("{:x}", Sha256::digest(bytes))) } + +#[cfg(target_os = "macos")] +pub(super) fn restore_generation_with_source_rebind( + engine: &Engine<'_>, + receipt: &Receipt, + selected: Option<&super::source_device_rebind::Selected>, +) -> Result { + use sha2::{Digest, Sha256}; + let original = restore_generation(engine, receipt)?; + let Some(selected) = selected else { + return Ok(original); + }; + let bytes = serde_json::to_vec(&( + "hack-graph-restore-source-device-rebind-v1", + original, + selected.raw_sha256(), + )) + .map_err(|_| error("graph_receipt", "Restore selection unavailable."))?; + Ok(format!("{:x}", Sha256::digest(bytes))) +} fn restore_inputs( candidate: &Candidate, options: RunOptions<'_>, @@ -198,9 +229,28 @@ fn restore_inputs( )); } let (mut receipt, root) = load(candidate, &engine, options.run_id)?; + #[cfg(target_os = "macos")] + let source_rebind = super::source_device_rebind::select(&engine, &receipt, &root)?; + #[cfg(target_os = "macos")] + if source_rebind.is_some() && fresh.is_none() { + return Err(error( + "graph_source_device_rebind", + "Source-device continuity requires a new foreground restore owner.", + )); + } let hostname_change = if let Some(fresh) = &fresh { - let change = - hostname_change::prepare(&engine, &inputs.review.plan, &receipt, &fresh.identity)?; + #[cfg(target_os = "macos")] + let source_receipt = source_rebind + .as_ref() + .map_or(&receipt, |selected| selected.source_receipt()); + #[cfg(not(target_os = "macos"))] + let source_receipt = &receipt; + let change = hostname_change::prepare( + &engine, + &inputs.review.plan, + source_receipt, + &fresh.identity, + )?; if change.is_some() && (!options.shared_source || options.live_source @@ -216,6 +266,10 @@ fn restore_inputs( None }; if let Some(fresh) = &fresh { + #[cfg(target_os = "macos")] + let generation = + restore_generation_with_source_rebind(&engine, &receipt, source_rebind.as_ref())?; + #[cfg(not(target_os = "macos"))] let generation = restore_generation(&engine, &receipt)?; if let Some(change) = &hostname_change { verify_fresh_change( @@ -256,11 +310,17 @@ fn restore_inputs( )); } super::super::source_job::check_reservations(&engine)?; + #[cfg(target_os = "macos")] + let source_receipt = source_rebind + .as_ref() + .map_or(&receipt, |selected| selected.source_receipt()); + #[cfg(not(target_os = "macos"))] + let source_receipt = &receipt; let ordinary_source = if hostname_change.is_none() { source::prepare_replay( &engine, &inputs, - &receipt, + source_receipt, options.source_revision, options.live_source, options.shared_source, @@ -361,7 +421,12 @@ fn restore_inputs( } if let Some(fresh) = &fresh { check_environment_deadline(fresh.deadline)?; - if restore_generation(&engine, &receipt)? != fresh.generation { + #[cfg(target_os = "macos")] + let generation = + restore_generation_with_source_rebind(&engine, &receipt, source_rebind.as_ref())?; + #[cfg(not(target_os = "macos"))] + let generation = restore_generation(&engine, &receipt)?; + if generation != fresh.generation { return Err(error( "graph_restore_refused", "Retained restore selection changed before effects.", @@ -374,7 +439,15 @@ fn restore_inputs( // Retain the complete acknowledged old generation before replacing its // boot-bound dependency owner. Historical cleanup is verified in its own context. if fresh.is_some() { + #[cfg(target_os = "macos")] + if let Some(selected) = &source_rebind { + selected.reverify(&engine)?; + } super::restore_history::retain(&root, &receipt)?; + #[cfg(target_os = "macos")] + if let Some(selected) = &source_rebind { + selected.apply_to_new_attempt(&mut receipt)?; + } receipt.relay_startup = None; receipt.relay_cleanup = None; } else { @@ -435,6 +508,10 @@ fn restore_inputs( session.fault_pause("restore-intent")?; } let result = (|| { + #[cfg(target_os = "macos")] + if let Some(selected) = &source_rebind { + selected.reverify(&session.engine)?; + } if fresh.is_some() && let Some(driver) = session.startup.as_mut() { diff --git a/packages/runtime-core/src/provider/graph/source.rs b/packages/runtime-core/src/provider/graph/source.rs index cd8260efb..c63626321 100644 --- a/packages/runtime-core/src/provider/graph/source.rs +++ b/packages/runtime-core/src/provider/graph/source.rs @@ -75,10 +75,18 @@ fn shared_paths( )?; selected.insert(mount.source.clone(), path); } - engine.guest().execute("set -eu; test \"$(findmnt -n -o FSTYPE --mountpoint \"$1\")\" = virtiofs; case \",$(findmnt -n -o OPTIONS --mountpoint \"$1\"),\" in *,rw,*) ;; *) exit 1;; esac", &[&share.guest_path], None)?; + verify_shared_mount(engine, share)?; Ok(selected) } +pub(super) fn verify_shared_mount( + engine: &Engine<'_>, + share: &super::super::ProjectShareIntent, +) -> Result<(), CandidateError> { + engine.guest().execute("set -eu; test \"$(findmnt -n -o FSTYPE --mountpoint \"$1\")\" = virtiofs; case \",$(findmnt -n -o OPTIONS --mountpoint \"$1\"),\" in *,rw,*) ;; *) exit 1;; esac", &[&share.guest_path], None)?; + Ok(()) +} + pub(super) fn requested(plan: &PlanData, revision: Option<&str>) -> Result<(), CandidateError> { if (mounts(plan).next().is_some() || plan diff --git a/packages/runtime-core/src/provider/graph/source_device_rebind.rs b/packages/runtime-core/src/provider/graph/source_device_rebind.rs new file mode 100644 index 000000000..c62e4d8d3 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/source_device_rebind.rs @@ -0,0 +1,642 @@ +//! One explicit source-device translation for an acknowledged stopped graph. +//! +//! The stopped receipt and its cleanup proofs remain immutable. This witness may +//! project only the share device for source admission into the next generation. +use super::{ + Candidate, CandidateError, Engine, Kind, Receipt, cleanup_enrollment, directory, foreground, + host_pin_recovery, inspect_resource, load, restore, source, state, +}; +use crate::provider::{ProjectShareIntent, lifecycle}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + fs::{self, File, OpenOptions}, + io::{Read, Seek, SeekFrom}, + os::unix::fs::{MetadataExt, OpenOptionsExt}, + path::{Path, PathBuf}, +}; + +const FILE: &str = "source-device-rebind.json"; +const LIMIT: u64 = 256 * 1024; +const STATE_LIMIT: u64 = 2 * 1024 * 1024; + +fn refused() -> CandidateError { + CandidateError::new( + "graph_source_device_rebind", + "Selected source-device continuity or retained graph identity changed; original receipts and data were preserved.", + ) +} + +fn sha256(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +fn absent(path: &Path) -> Result { + match fs::symlink_metadata(path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true), + Err(_) => Err(refused()), + Ok(_) => Ok(false), + } +} + +pub(super) fn require_no_pending(candidate: &Candidate, run: &str) -> Result<(), CandidateError> { + if absent( + &directory(candidate, run)? + .join(FILE) + .with_extension("pending"), + )? { + Ok(()) + } else { + Err(refused()) + } +} + +fn no_pending(root: &Path) -> Result<(), CandidateError> { + for name in [ + "state.pending", + "absent-publication-cleanup.pending", + "absent-publication-retirement.pending", + "source-device-rebind.pending", + ] { + if !absent(&root.join(name))? { + return Err(refused()); + } + } + Ok(()) +} + +fn encoded(value: &impl Serialize) -> Result, CandidateError> { + serde_json::to_vec_pretty(value).map_err(|_| refused()) +} + +struct PinnedRaw { + file: File, + bytes: Vec, + file_id: (u64, u64), + parent_id: (u64, u64), + path: PathBuf, +} + +impl PinnedRaw { + fn reverify(&self) -> Result<(), CandidateError> { + let parent = self.path.parent().ok_or_else(refused)?; + state::check_private_directory(parent).map_err(|_| refused())?; + let parent_meta = fs::symlink_metadata(parent).map_err(|_| refused())?; + let meta = self.file.metadata().map_err(|_| refused())?; + let path_meta = fs::symlink_metadata(&self.path).map_err(|_| refused())?; + // SAFETY: geteuid has no arguments or side effects. + let uid = unsafe { libc::geteuid() }; + if (parent_meta.dev(), parent_meta.ino()) != self.parent_id + || (meta.dev(), meta.ino()) != self.file_id + || (path_meta.dev(), path_meta.ino()) != self.file_id + || !meta.is_file() + || meta.nlink() != 1 + || meta.uid() != uid + || path_meta.uid() != uid + || meta.mode() & 0o7777 != 0o600 + || path_meta.mode() & 0o7777 != 0o600 + || path_meta.nlink() != 1 + || meta.len() != self.bytes.len() as u64 + { + return Err(refused()); + } + let mut file = &self.file; + file.seek(SeekFrom::Start(0)).map_err(|_| refused())?; + let mut bytes = Vec::new(); + file.take(self.bytes.len() as u64 + 1) + .read_to_end(&mut bytes) + .map_err(|_| refused())?; + if bytes != self.bytes { + return Err(refused()); + } + Ok(()) + } +} + +fn pin_raw(path: &Path, limit: u64, allow_empty: bool) -> Result { + let parent = path.parent().ok_or_else(refused)?; + state::check_private_directory(parent).map_err(|_| refused())?; + let parent_meta = fs::symlink_metadata(parent).map_err(|_| refused())?; + let mut file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path) + .map_err(|_| refused())?; + let before = file.metadata().map_err(|_| refused())?; + // SAFETY: geteuid has no arguments or side effects. + if !before.is_file() + || before.nlink() != 1 + || before.uid() != unsafe { libc::geteuid() } + || before.mode() & 0o7777 != 0o600 + || before.len() > limit + || (!allow_empty && before.len() == 0) + { + return Err(refused()); + } + let mut bytes = Vec::new(); + file.by_ref() + .take(limit + 1) + .read_to_end(&mut bytes) + .map_err(|_| refused())?; + file.seek(SeekFrom::Start(0)).map_err(|_| refused())?; + let mut confirmation = Vec::new(); + file.by_ref() + .take(limit + 1) + .read_to_end(&mut confirmation) + .map_err(|_| refused())?; + let after = file.metadata().map_err(|_| refused())?; + let path_after = fs::symlink_metadata(path).map_err(|_| refused())?; + let parent_after = fs::symlink_metadata(parent).map_err(|_| refused())?; + if bytes != confirmation + || bytes.len() as u64 != before.len() + || [after.dev(), path_after.dev()] != [before.dev(); 2] + || [after.ino(), path_after.ino()] != [before.ino(); 2] + || [after.len(), path_after.len()] != [before.len(); 2] + || [after.mtime(), path_after.mtime()] != [before.mtime(); 2] + || [after.mtime_nsec(), path_after.mtime_nsec()] != [before.mtime_nsec(); 2] + || [after.ctime(), path_after.ctime()] != [before.ctime(); 2] + || [after.ctime_nsec(), path_after.ctime_nsec()] != [before.ctime_nsec(); 2] + || path_after.nlink() != 1 + || (parent_after.dev(), parent_after.ino()) != (parent_meta.dev(), parent_meta.ino()) + { + return Err(refused()); + } + Ok(PinnedRaw { + file, + bytes, + file_id: (before.dev(), before.ino()), + parent_id: (parent_meta.dev(), parent_meta.ino()), + path: path.to_owned(), + }) +} + +fn raw( + path: &Path, + limit: u64, + allow_empty: bool, +) -> Result<(Vec, (u64, u64)), CandidateError> { + let pin = pin_raw(path, limit, allow_empty)?; + Ok((pin.bytes, pin.file_id)) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Witness { + version: u8, + run: String, + owner: String, + namespace: String, + plan: String, + original_ready_sha256: String, + stopped_raw_sha256: String, + absent_intent_raw_sha256: String, + retirement_raw_sha256: String, + original_owner_raw_sha256: String, + current_owner_raw_sha256: String, + host_boot_micros: u64, + previous_guest_boot: String, + current_guest_boot: String, + old_share: ProjectShareIntent, + current_share: ProjectShareIntent, + retained_volume_projections: BTreeMap, + retained_volumes: BTreeMap, +} + +impl Witness { + fn project(&self, receipt: &Receipt) -> Result { + let mut projected = receipt.clone(); + let shared = projected + .source + .as_mut() + .and_then(|source| source.shared.as_mut()) + .ok_or_else(refused)?; + if *shared != self.old_share { + return Err(refused()); + } + shared.device = self.current_share.device; + if *shared != self.current_share { + return Err(refused()); + } + Ok(projected) + } + + fn unchanged_identity(&self, receipt: &Receipt) -> bool { + self.version == 1 + && self.run == receipt.run + && self.owner == receipt.owner + && self.namespace == receipt.namespace + && self.plan == receipt.plan_id + && self.old_share.device != self.current_share.device + && self.old_share.project == self.current_share.project + && self.old_share.guest_path == self.current_share.guest_path + && self.old_share.inode == self.current_share.inode + && self.old_share.unfiltered_source == self.current_share.unfiltered_source + && receipt.source.as_ref().and_then(|s| s.shared.as_ref()) == Some(&self.old_share) + } +} + +/// Pinned complete witness for the original stopped receipt only. A subsequent +/// graph generation cannot use this object as cleanup or source authority. +pub(super) struct Selected { + witness: Witness, + pin: PinnedRaw, + original: Receipt, + projected: Receipt, + root: PathBuf, +} + +impl Selected { + pub(super) fn raw_sha256(&self) -> String { + sha256(&self.pin.bytes) + } + + pub(super) fn source_receipt(&self) -> &Receipt { + &self.projected + } + + pub(super) fn apply_to_new_attempt(&self, receipt: &mut Receipt) -> Result<(), CandidateError> { + if encoded(receipt)? != encoded(&self.original)? { + return Err(refused()); + } + receipt.source = self.projected.source.clone(); + Ok(()) + } + + /// Recheck raw bytes, pathname identity, Owner, guest and volumes before the + /// first new-attempt effect/write. The caller still holds the Engine lease. + pub(super) fn reverify(&self, engine: &Engine<'_>) -> Result<(), CandidateError> { + self.pin.reverify()?; + validate_current(engine, &self.root, &self.original, &self.witness)?; + let current: Receipt = state::read(&self.root.join("state.json"))?; + if encoded(¤t)? != encoded(&self.original)? { + return Err(refused()); + } + Ok(()) + } +} + +fn validate_current( + engine: &Engine<'_>, + root: &Path, + receipt: &Receipt, + witness: &Witness, +) -> Result<(), CandidateError> { + no_pending(root)?; + if receipt.phase != "stopped-data-retained" || !witness.unchanged_identity(receipt) { + return Err(refused()); + } + let state_bytes = raw(&root.join("state.json"), STATE_LIMIT, false)?.0; + if state_bytes != encoded(receipt)? || sha256(&state_bytes) != witness.stopped_raw_sha256 { + return Err(refused()); + } + for (name, expected) in [ + ( + "absent-publication-cleanup.json", + &witness.absent_intent_raw_sha256, + ), + ( + "absent-publication-retirement.json", + &witness.retirement_raw_sha256, + ), + ] { + if sha256(&raw(&root.join(name), STATE_LIMIT, false)?.0) != *expected { + return Err(refused()); + } + } + let candidate = engine.guest().candidate(); + let owner_root = candidate.state_root.join("run/smolvm"); + if sha256(&raw(&owner_root.join("owner.json"), STATE_LIMIT, false)?.0) + != witness.current_owner_raw_sha256 + || lifecycle::host_filesystem::host_boot_micros()? != witness.host_boot_micros + || engine.guest().boot_id() != witness.current_guest_boot + || engine.guest().project_share() != Some(&witness.current_share) + { + return Err(refused()); + } + witness.current_share.validate().map_err(|_| refused())?; + source::verify_shared_mount(engine, &witness.current_share).map_err(|_| refused())?; + if restore::observed_volumes(engine, receipt)? != witness.retained_volumes { + return Err(refused()); + } + host_pin_recovery::verify_volume_projections( + engine, + receipt, + &witness.retained_volume_projections, + ) + .map_err(|_| refused())?; + for resource in receipt.resources.values() { + if resource.kind != Kind::Volume { + let observed = inspect_resource(engine, receipt, resource)?; + let mut by_name = resource.clone(); + by_name.id = None; + if resource.phase != "absent" + || observed.is_some() + || inspect_resource(engine, receipt, &by_name)?.is_some() + { + return Err(refused()); + } + } + } + cleanup_enrollment::retention(root, receipt)?; + Ok(()) +} + +/// A committed witness applies only to its original stopped state. Once the +/// next generation has a current share it remains audit data, never authority. +pub(super) fn select( + engine: &Engine<'_>, + receipt: &Receipt, + root: &Path, +) -> Result, CandidateError> { + if !absent(&root.join(FILE).with_extension("pending"))? { + return Err(refused()); + } + let path = root.join(FILE); + if absent(&path)? { + return Ok(None); + } + let pin = pin_raw(&path, LIMIT, false)?; + let witness: Witness = serde_json::from_slice(&pin.bytes).map_err(|_| refused())?; + if pin.bytes != encoded(&witness)? { + return Err(refused()); + } + let state_sha = sha256(&raw(&root.join("state.json"), STATE_LIMIT, false)?.0); + if state_sha != witness.stopped_raw_sha256 { + // The witness is historical only when normal source ownership has moved on. + if receipt.owner == witness.owner + && receipt.run == witness.run + && receipt.namespace == witness.namespace + && receipt.source.as_ref().and_then(|s| s.shared.as_ref()) + == engine.guest().project_share() + { + return Ok(None); + } + return Err(refused()); + } + validate_current(engine, root, receipt, &witness)?; + let projected = witness.project(receipt)?; + Ok(Some(Selected { + witness, + pin, + original: receipt.clone(), + projected, + root: root.to_owned(), + })) +} + +fn selected_witness( + candidate: &Candidate, + run: &str, + retired: &foreground::transport::Retired, + engine: &Engine<'_>, +) -> Result<(Witness, Receipt, PathBuf), CandidateError> { + let (receipt, root) = load(candidate, engine, run)?; + let proof = super::absent_publication_cleanup::verify_completed_under( + candidate, engine, run, &receipt, retired, + )? + .ok_or_else(refused)?; + let old_share = proof.old_share.ok_or_else(refused)?; + let current_share = proof.current_share.ok_or_else(refused)?; + let state_bytes = raw(&root.join("state.json"), STATE_LIMIT, false)?.0; + if state_bytes != encoded(&receipt)? + || receipt.phase != "stopped-data-retained" + || sha256(&state_bytes) != proof.completed_stopped_sha256 + || engine.guest().project_share() != Some(¤t_share) + || lifecycle::host_filesystem::host_boot_micros()? != proof.host_boot_micros + || engine.guest().boot_id() != proof.current_guest_boot + { + return Err(refused()); + } + let witness = Witness { + version: 1, + run: receipt.run.clone(), + owner: receipt.owner.clone(), + namespace: receipt.namespace.clone(), + plan: receipt.plan_id.clone(), + original_ready_sha256: proof.original_ready_sha256, + stopped_raw_sha256: proof.completed_stopped_sha256, + absent_intent_raw_sha256: proof.intent_raw_sha256, + retirement_raw_sha256: proof.retirement_raw_sha256, + original_owner_raw_sha256: proof.original_owner_sha256, + current_owner_raw_sha256: proof.current_owner_sha256, + host_boot_micros: proof.host_boot_micros, + previous_guest_boot: proof.previous_guest_boot, + current_guest_boot: proof.current_guest_boot, + old_share, + current_share, + retained_volume_projections: proof.retained_volumes, + retained_volumes: restore::observed_volumes(engine, &receipt)?, + }; + validate_current(engine, &root, &receipt, &witness)?; + retired.verify()?; + Ok((witness, receipt, root)) +} + +fn selected_output(witness: &Witness, committed: bool) -> Result { + Ok(json!({ + "run": witness.run, + "owner": witness.owner, + "namespace": witness.namespace, + "plan": witness.plan, + "stopped_receipt_sha256": witness.stopped_raw_sha256, + "selection_sha256": sha256(&encoded(witness)?), + "committed": committed, + "qualification": "explicit-legacy-migration-original-volume-continuity-unproven", + })) +} + +/// Read-only selection holds the retired publisher lock before the Engine lease. +pub fn inspect(candidate: &Candidate, run: &str) -> Result { + let retired = foreground::transport::Retired::acquire(candidate, run)?.ok_or_else(refused)?; + let engine = Engine::connect_cleanup_wait(candidate)?; + let (witness, receipt, root) = selected_witness(candidate, run, &retired, &engine)?; + let committed = if absent(&root.join(FILE))? { + false + } else { + let selected = select(&engine, &receipt, &root)?.ok_or_else(refused)?; + selected.witness == witness + }; + if !absent(&root.join(FILE).with_extension("pending"))? + || !committed && !absent(&root.join(FILE))? + { + return Err(refused()); + } + retired.verify()?; + selected_output(&witness, committed) +} + +/// Publish one immutable selected device-only source witness; no graph receipt +/// or cleanup/retirement digest is rewritten. +pub fn recover(candidate: &Candidate, run: &str, expected: &str) -> Result { + if !super::hex(expected, 64) { + return Err(refused()); + } + let retired = foreground::transport::Retired::acquire(candidate, run)?.ok_or_else(refused)?; + let engine = Engine::connect_cleanup_wait(candidate)?; + require_no_pending(candidate, run)?; + let (witness, receipt, root) = selected_witness(candidate, run, &retired, &engine)?; + let bytes = encoded(&witness)?; + if bytes.len() as u64 > LIMIT || sha256(&bytes) != expected { + return Err(refused()); + } + let path = root.join(FILE); + if !absent(&path)? { + let selected = select(&engine, &receipt, &root)?.ok_or_else(refused)?; + if selected.pin.bytes != bytes { + return Err(refused()); + } + return selected_output(&witness, true); + } + retired.verify()?; + validate_current(&engine, &root, &receipt, &witness)?; + state::write(&path, &witness)?; + let selected = select(&engine, &receipt, &root)?.ok_or_else(refused)?; + if selected.pin.bytes != bytes { + return Err(refused()); + } + retired.verify()?; + selected_output(&witness, true) +} + +#[cfg(all(test, feature = "environment-launcher"))] +pub(in crate::provider::graph) fn fixture_wrong_volume_projection(bytes: &[u8]) -> Vec { + let mut witness: Witness = serde_json::from_slice(bytes).unwrap(); + witness + .retained_volume_projections + .get_mut("volume:data") + .unwrap()["observed_created_at"] = json!("foreign"); + encoded(&witness).unwrap() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + fn private_file(path: &Path, bytes: &[u8]) { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) + .unwrap(); + use std::io::Write; + file.write_all(bytes).unwrap(); + } + + #[test] + fn pinned_witness_refuses_equal_bytes_at_a_replacement_path_or_parent() { + let fixture = super::super::tests::Fixture::new(); + let parent = fixture.0.join("witness"); + fs::create_dir(&parent).unwrap(); + fs::set_permissions(&parent, fs::Permissions::from_mode(0o700)).unwrap(); + let path = parent.join(FILE); + private_file(&path, b"proof"); + let pin = pin_raw(&path, LIMIT, false).unwrap(); + pin.reverify().unwrap(); + + fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); + assert!(pin.reverify().is_err()); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + pin.reverify().unwrap(); + + fs::rename(&path, parent.join("archived")).unwrap(); + private_file(&path, b"proof"); + assert!(pin.reverify().is_err()); + + fs::rename(&parent, fixture.0.join("old-witness")).unwrap(); + fs::create_dir(&parent).unwrap(); + fs::set_permissions(&parent, fs::Permissions::from_mode(0o700)).unwrap(); + private_file(&path, b"proof"); + assert!(pin.reverify().is_err()); + } + + #[test] + fn pending_witness_is_preserved_and_refuses_publication() { + let fixture = super::super::tests::Fixture::new(); + let candidate = Candidate::discover(&fixture.0).unwrap(); + let run = "a".repeat(32); + let root = directory(&candidate, &run).unwrap(); + state::private_directory(&root).unwrap(); + let pending = root.join(FILE).with_extension("pending"); + private_file(&pending, b""); + assert!(require_no_pending(&candidate, &run).is_err()); + assert!(no_pending(&root).is_err()); + assert_eq!(fs::read(&pending).unwrap(), b""); + fs::remove_file(&pending).unwrap(); + private_file(&pending, b"foreign"); + assert!(require_no_pending(&candidate, &run).is_err()); + assert!(no_pending(&root).is_err()); + assert_eq!(fs::read(&pending).unwrap(), b"foreign"); + } + + #[test] + fn source_projection_changes_only_the_selected_device() { + let old_share = ProjectShareIntent { + project: PathBuf::from("/private/tmp/example-project"), + guest_path: "/mnt/hack-projects/exact".into(), + device: 17, + inode: 23, + unfiltered_source: true, + }; + let mut current_share = old_share.clone(); + current_share.device = 19; + let receipt: Receipt = serde_json::from_value(json!({ + "version": 1, + "run": "a".repeat(32), + "owner": "b".repeat(32), + "namespace": "c".repeat(64), + "plan_id": "d".repeat(64), + "phase": "stopped-data-retained", + "readiness": {}, + "resources": {}, + "source": { + "shared": old_share, + "revision": "e".repeat(64), + "archive_sha256": "f".repeat(64), + "selection_sha256": "0".repeat(64), + }, + })) + .unwrap(); + let witness = Witness { + version: 1, + run: receipt.run.clone(), + owner: receipt.owner.clone(), + namespace: receipt.namespace.clone(), + plan: receipt.plan_id.clone(), + original_ready_sha256: "1".repeat(64), + stopped_raw_sha256: "2".repeat(64), + absent_intent_raw_sha256: "3".repeat(64), + retirement_raw_sha256: "4".repeat(64), + original_owner_raw_sha256: "5".repeat(64), + current_owner_raw_sha256: "6".repeat(64), + host_boot_micros: 7, + previous_guest_boot: "old".into(), + current_guest_boot: "new".into(), + old_share, + current_share, + retained_volume_projections: BTreeMap::new(), + retained_volumes: BTreeMap::new(), + }; + assert!(witness.unchanged_identity(&receipt)); + let projected = witness.project(&receipt).unwrap(); + let mut expected = receipt.clone(); + expected + .source + .as_mut() + .unwrap() + .shared + .as_mut() + .unwrap() + .device = 19; + assert_eq!(encoded(&projected).unwrap(), encoded(&expected).unwrap()); + assert_eq!(receipt.source.unwrap().shared.unwrap().device, 17); + + let mut foreign = witness.clone(); + foreign.current_share.inode += 1; + assert!(!foreign.unchanged_identity(&expected)); + assert!(foreign.project(&expected).is_err()); + } +} diff --git a/packages/runtime-core/tests/source_device_rebind_cli_contract.rs b/packages/runtime-core/tests/source_device_rebind_cli_contract.rs new file mode 100644 index 000000000..d50bd340a --- /dev/null +++ b/packages/runtime-core/tests/source_device_rebind_cli_contract.rs @@ -0,0 +1,62 @@ +//! The explicit continuity acknowledgement must be exact before provider access. +use serde_json::Value; +use std::{fs, path::Path, process::Command}; + +#[test] +fn source_device_rebind_requires_exact_selection_and_acknowledgement() { + let checkout = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .unwrap(); + let home = std::env::temp_dir().join(format!( + "hack-source-rebind-cli-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&home).unwrap(); + let full = [ + "graph".to_owned(), + "recover-source-device-rebind".to_owned(), + "--run-id".to_owned(), + "a".repeat(32), + "--expect-selection".to_owned(), + "b".repeat(64), + "--accept-legacy-device-rebind".to_owned(), + "--json".to_owned(), + ]; + let mut cases = Vec::new(); + for flag in ["--accept-legacy-device-rebind", "--json"] { + let mut duplicate = full.to_vec(); + duplicate.push(flag.to_owned()); + cases.push(duplicate); + } + for flag in ["--run-id", "--expect-selection"] { + let mut missing = full.to_vec(); + let index = missing.iter().position(|value| value == flag).unwrap(); + missing.drain(index..index + 2); + cases.push(missing); + } + let mut unacknowledged = full.to_vec(); + unacknowledged.retain(|value| value != "--accept-legacy-device-rebind"); + cases.push(unacknowledged); + for arguments in cases { + let output = Command::new(env!("CARGO_BIN_EXE_hack-runtime-candidate")) + .arg("--candidate-root") + .arg(&checkout) + .args(&arguments) + .env_clear() + .env("PATH", "/nonexistent") + .env("HOME", &home) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2), "{arguments:?}"); + assert!(output.stdout.is_empty()); + let failure: Value = serde_json::from_slice(&output.stderr).unwrap(); + assert_eq!(failure["code"], "graph_arguments", "{arguments:?}"); + assert_eq!(fs::read_dir(&home).unwrap().count(), 0); + } + fs::remove_dir(&home).unwrap(); +} From b84385759d1f3426d52b9719cbaa189aad157175 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 04:16:08 -0400 Subject: [PATCH 11/25] test(runtime): isolate publisher acknowledgement transport --- .../runtime-core/tests/stream_relay_contract.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/runtime-core/tests/stream_relay_contract.rs b/packages/runtime-core/tests/stream_relay_contract.rs index 8ff435c2a..06bc03779 100644 --- a/packages/runtime-core/tests/stream_relay_contract.rs +++ b/packages/runtime-core/tests/stream_relay_contract.rs @@ -773,15 +773,16 @@ fn publisher_refuses_port_conflicts_and_stale_reused_upstream() { #[test] fn publisher_requires_ack_before_forwarding_and_preserves_upstream_socket() { - use std::os::unix::net::UnixListener; + use std::os::unix::{fs::MetadataExt, net::UnixListener}; let root = PathBuf::from(format!("/tmp/hkp-ack-{}", std::process::id())); fs::create_dir(&root).unwrap(); let path = root.join("socket"); let upstream = UnixListener::bind(&path).unwrap(); fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap(); let token = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - let port = free_loopback_port(); - let mut publication = publisher(&path, token, port); + let owned = fs::symlink_metadata(&path).unwrap(); + // The ack contract is transport-independent; avoid releasing a TCP port before spawn. + let mut publication = frontend_publisher(&path, token, 0, None, true); let worker = thread::spawn(move || { let (mut socket, _) = upstream.accept().unwrap(); socket @@ -795,7 +796,7 @@ fn publisher_requires_ack_before_forwarding_and_preserves_upstream_socket() { socket.read_to_end(&mut extra).unwrap(); assert!(extra.is_empty()); }); - let mut client = std::net::TcpStream::connect(("127.0.0.1", port)).unwrap(); + let mut client = publication.connect(); client .set_read_timeout(Some(Duration::from_secs(3))) .unwrap(); @@ -808,7 +809,9 @@ fn publisher_requires_ack_before_forwarding_and_preserves_upstream_socket() { } worker.join().unwrap(); publication.stop(); - assert!(path.exists()); + let retained = fs::symlink_metadata(&path).unwrap(); + assert_eq!((retained.dev(), retained.ino()), (owned.dev(), owned.ino())); + assert_eq!(retained.mode(), owned.mode()); fs::remove_dir_all(root).unwrap(); } From 015e00dbf4f8535ffa66473fca57ce5576bd05f1 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 04:18:16 -0400 Subject: [PATCH 12/25] test(runtime): qualify bind-only source recovery invocation --- .../absent_publication_recovery.rs | 19 +++++++++++--- .../native_test/source_device_rebind.rs | 26 ------------------- 2 files changed, 15 insertions(+), 30 deletions(-) diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs b/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs index fb579a577..9c4061fea 100644 --- a/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs +++ b/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs @@ -11,10 +11,21 @@ use std::{ pub(super) fn ready(owner: &mut Process, run: &str, deadline: Instant) { loop { - assert!( - owner.poll().is_none(), - "foreground owner exited before ready" - ); + if let Some(status) = owner.poll() { + let error: Option = serde_json::from_slice(&owner.err).ok(); + let code = error + .as_ref() + .and_then(|value| value["code"].as_str()) + .unwrap_or("unstructured"); + let cause_code = error + .as_ref() + .and_then(|value| value["cause_code"].as_str()) + .unwrap_or("none"); + panic!( + "foreground owner exited before ready: status={:?}, code={code}, cause_code={cause_code}", + status.code() + ); + } if let Some(end) = owner.out.iter().position(|byte| *byte == b'\n') { let value: Value = serde_json::from_slice(&owner.out[..end]).unwrap(); assert_eq!(value["kind"], "graph_foreground_ready"); diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test/source_device_rebind.rs b/packages/runtime-core/src/provider/graph/foreground/native_test/source_device_rebind.rs index 7c4129c54..a44d6326c 100644 --- a/packages/runtime-core/src/provider/graph/foreground/native_test/source_device_rebind.rs +++ b/packages/runtime-core/src/provider/graph/foreground/native_test/source_device_rebind.rs @@ -146,30 +146,6 @@ fn selected_source_device_rebind_restores_same_run_and_two_ordinary_generations( ) .unwrap(); let normalized = project.join("compose.yaml"); - let publication = checked_cli( - "publish-source", - &binary, - &candidate, - &[ - "project", - "publish-source", - "--project", - project.to_str().unwrap(), - "--file", - "compose.yaml", - "--expect-plan", - &review.plan_id, - "--normalized-file", - normalized.to_str().unwrap(), - "--expect-original", - &review.plan.compose_sha256, - "--expect-namespace", - &review.plan.namespace, - "--json", - ], - deadline, - ); - let revision = publication["revision"].as_str().unwrap(); let dependencies_path = fixture.0.join("dependencies.json"); state::write( &dependencies_path, @@ -215,8 +191,6 @@ fn selected_source_device_rebind_restores_same_run_and_two_ordinary_generations( dependencies_path.to_str().unwrap(), "--expect-dependencies", dependency["dependency_plan_id"].as_str().unwrap(), - "--source-revision", - revision, "--shared-source", "--normalized-file", normalized.to_str().unwrap(), From 0f3b971b7d3b814d2246f3783fe1ab1b46fe66d7 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 04:45:48 -0400 Subject: [PATCH 13/25] fix(runtime): prefer verified current cleanup over historical enrollment --- packages/runtime-core/README.md | 5 + .../graph/absent_publication_cleanup.rs | 179 ++++++++++++++++++ .../src/provider/graph/cleanup_enrollment.rs | 3 +- .../src/provider/graph/live_owner_cleanup.rs | 34 ++++ 4 files changed, 220 insertions(+), 1 deletion(-) diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index cac696a41..15886c72a 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -820,6 +820,11 @@ identities are checked independently. The explicit acceptance records that legacy receipts cannot establish original physical volume continuity across the host reboot. +An exact completed cleanup and retirement for the current stopped receipt takes +precedence over older cleanup records. Pending operations, unresolved initializer +effects and mismatched receipt or retirement identities still refuse recovery; +historical records cannot authorize a later generation. + `restore-selection` then binds the witness's raw hash to the selected generation. The source checks use only an in-memory device projection; original retention and restore history consume the unchanged stopped receipt. The new receipt receives diff --git a/packages/runtime-core/src/provider/graph/absent_publication_cleanup.rs b/packages/runtime-core/src/provider/graph/absent_publication_cleanup.rs index ead3beec2..df5007fa4 100644 --- a/packages/runtime-core/src/provider/graph/absent_publication_cleanup.rs +++ b/packages/runtime-core/src/provider/graph/absent_publication_cleanup.rs @@ -847,6 +847,32 @@ pub(super) fn publication_allowed( Ok(()) } +/// Prefer only an exact current completion over historical cleanup sidecars. +/// A later generation must still pass the ordinary enrollment checks. +pub(super) fn retained_current(root: &Path, receipt: &Receipt) -> Result { + let Some(intent) = read_intent(root)? else { + return Ok(false); + }; + no_pending(root)?; + require_absent(&root.join("live-owner-cleanup.pending"))?; + super::initializer_cache::require_resolved(receipt)?; + let requested = digest(&serde_json::to_vec_pretty(receipt).map_err(|_| refused())?); + if intent.complete_sha256.as_deref() != Some(requested.as_str()) { + return Ok(false); + } + if !intent.selection.matches_graph(receipt) || receipt.relay_cleanup.is_some() { + return Err(refused()); + } + let bridges: super::bridges::cleanup::Selection = + state::read_bounded(&root.join("relay-cleanup-bridges.json"), 65536)?; + if bridges != intent.bridges { + return Err(refused()); + } + dead_owner_cleanup::require_historical_recovery(root, receipt)?; + super::live_owner_cleanup::require_historical_recovery(root, receipt)?; + retained(root, receipt) +} + /// Retention is bound to the completed stopped receipt, not to missing paths. pub(super) fn retained(root: &Path, receipt: &Receipt) -> Result { let Some(intent) = read_intent(root)? else { @@ -1163,6 +1189,159 @@ mod tests { (fixture, candidate) } + #[test] + fn current_absent_completion_precedes_historical_enrollment_but_never_partial_proofs() { + let (fixture, _) = candidate(); + let root = &fixture.0; + let original: Receipt = serde_json::from_value(json!({ + "version":1,"run":"a".repeat(32),"owner":"b".repeat(32), + "namespace":"c".repeat(64),"plan_id":"d".repeat(64), + "phase":"ready-observed","readiness":{},"resources":{"container:web":{ + "kind":"container","key":"web","name":"owned-web","id":"b".repeat(64),"image":null,"phase":"absent"}}, + "relay_startup":{"control_only":true,"guest_root":null, + "control_root":"/private/control","artifact":"e".repeat(64),"services":{}} + })) + .unwrap(); + let mut current = original.clone(); + current.phase = "stopped-data-retained".into(); + let original_hash = digest(&serde_json::to_vec_pretty(&original).unwrap()); + let complete = digest(&serde_json::to_vec_pretty(¤t).unwrap()); + let selection: Selection = serde_json::from_value(json!({ + "version":1,"candidate":root,"run":current.run,"owner":current.owner, + "namespace":current.namespace,"plan":current.plan_id, + "original_owner_path":"/private/owner","original_owner_sha256":"1".repeat(64), + "filesystem_inspection_path":"/private/inspection","filesystem_inspection_sha256":"2".repeat(64), + "current_owner_sha256":"3".repeat(64),"graph_sha256":original_hash, + "host_boot_micros":1,"old_device":2,"new_device":3, + "previous_guest_boot":"previous","current_guest_boot":"current", + "foreground_root":"/private/foreground","control_root":"/private/control", + "source_shared":null,"environment_inventory":{},"scoped_bridge_projection":{}, + "retained_volumes":{},"dependency_reservation":null,"qualification":"test" + })).unwrap(); + let intent = Intent { + version: 1, + selection_sha256: selection.digest().unwrap(), + selection, + original: original.clone(), + environment: json!({}), + bridges: serde_json::from_value(json!({"version":1,"owner":current.owner, + "boot":"current","run":current.run,"plan":current.plan_id, + "capacity":1,"serial":1,"selected":{}})) + .unwrap(), + prior_bridges: None, + complete_sha256: Some(complete.clone()), + }; + let retirement = Retirement { + version: 1, + selection_sha256: intent.selection_sha256.clone(), + complete_sha256: complete, + owner: current.owner.clone(), + run: current.run.clone(), + }; + state::write(&root.join("state.json"), ¤t).unwrap(); + state::write(&root.join(INTENT), &intent).unwrap(); + state::write(&root.join(RETIREMENT), &retirement).unwrap(); + state::write(&root.join("relay-cleanup-bridges.json"), &intent.bridges).unwrap(); + // A completed older recovery is retained as exact history, not current authority. + let mut prior = current.clone(); + prior.resources.get_mut("container:web").unwrap().id = Some("a".repeat(64)); + let mut prior_original = prior.clone(); + prior_original.phase = "ready-observed".into(); + let dead = json!({ + "version":1,"original_sha256":digest(&serde_json::to_vec_pretty(&prior_original).unwrap()), + "owner_sha256":"4".repeat(64),"old_boot":"older","new_boot":"previous", + "original":prior_original,"environment":null,"bridges":null, + "complete_sha256":digest(&serde_json::to_vec_pretty(&prior).unwrap()) + }); + state::write(&root.join("dead-owner-cleanup.json"), &dead).unwrap(); + state::write( + &root.join("restore-history.json"), + &json!({ + "version":1,"run":current.run,"owner":current.owner,"namespace":current.namespace, + "truncated":false,"entries":[prior],"legacy":{} + }), + ) + .unwrap(); + assert_eq!( + super::super::dead_owner_cleanup::retained(root, ¤t) + .unwrap_err() + .code, + "graph_relay_enrollment" + ); + assert!(super::super::cleanup_enrollment::retention(root, ¤t).is_ok()); + let live = json!({ + "version":1,"boot":"previous","original":prior_original, + "original_sha256":dead["original_sha256"],"foreground_sha256":"f".repeat(64), + "relay":{"bytes":[],"record_id":[1,1]},"environment":{},"bridges":intent.bridges, + "prior_bridges":null,"listeners_retired":true,"complete_sha256":dead["complete_sha256"] + }); + state::write(&root.join("live-owner-cleanup.json"), &live).unwrap(); + assert!(super::super::cleanup_enrollment::retention(root, ¤t).is_ok()); + for (name, valid) in [ + ("dead-owner-cleanup.json", &dead), + ("live-owner-cleanup.json", &live), + ] { + for field in ["complete_sha256", "original_sha256"] { + let mut invalid = valid.clone(); + invalid[field] = if field == "complete_sha256" { + Value::Null + } else { + json!("0".repeat(64)) + }; + state::write(&root.join(name), &invalid).unwrap(); + assert!(super::super::cleanup_enrollment::retention(root, ¤t).is_err()); + } + let mut invalid = valid.clone(); + invalid["original"]["owner"] = json!("0".repeat(32)); + state::write(&root.join(name), &invalid).unwrap(); + assert!(super::super::cleanup_enrollment::retention(root, ¤t).is_err()); + state::write(&root.join(name), valid).unwrap(); + } + let mut unretired = live.clone(); + unretired["listeners_retired"] = json!(false); + state::write(&root.join("live-owner-cleanup.json"), &unretired).unwrap(); + assert!(super::super::cleanup_enrollment::retention(root, ¤t).is_err()); + state::write(&root.join("live-owner-cleanup.json"), &live).unwrap(); + let history = fs::read(root.join("restore-history.json")).unwrap(); + fs::remove_file(root.join("restore-history.json")).unwrap(); + assert!(super::super::cleanup_enrollment::retention(root, ¤t).is_err()); + fs::write(root.join("restore-history.json"), history).unwrap(); + let mut changed_bridges = serde_json::to_value(&intent.bridges).unwrap(); + changed_bridges["serial"] = json!(99); + state::write(&root.join("relay-cleanup-bridges.json"), &changed_bridges).unwrap(); + assert!(super::super::cleanup_enrollment::retention(root, ¤t).is_err()); + state::write(&root.join("relay-cleanup-bridges.json"), &intent.bridges).unwrap(); + let before = fs::read(root.join("state.json")).unwrap(); + for pending in [ + "state.pending", + "dead-owner-cleanup.pending", + "live-owner-cleanup.pending", + "absent-publication-cleanup.pending", + "absent-publication-retirement.pending", + ] { + fs::write(root.join(pending), b"interrupted").unwrap(); + assert!(super::super::cleanup_enrollment::retention(root, ¤t).is_err()); + fs::remove_file(root.join(pending)).unwrap(); + } + for change in 0..4 { + let mut invalid = intent.clone(); + match change { + 0 => invalid.complete_sha256 = None, + 1 => invalid.complete_sha256 = Some("6".repeat(64)), + 2 => invalid.selection.owner = "7".repeat(32), + _ => invalid.selection.plan = "8".repeat(64), + } + state::write(&root.join(INTENT), &invalid).unwrap(); + assert!(super::super::cleanup_enrollment::retention(root, ¤t).is_err()); + } + state::write(&root.join(INTENT), &intent).unwrap(); + let mut invalid = retirement.clone(); + invalid.complete_sha256 = "9".repeat(64); + state::write(&root.join(RETIREMENT), &invalid).unwrap(); + assert!(super::super::cleanup_enrollment::retention(root, ¤t).is_err()); + assert_eq!(fs::read(root.join("state.json")).unwrap(), before); + } + #[test] fn read_only_inspection_of_existing_reservation_never_creates_a_root() { let (_fixture, candidate) = candidate(); diff --git a/packages/runtime-core/src/provider/graph/cleanup_enrollment.rs b/packages/runtime-core/src/provider/graph/cleanup_enrollment.rs index 665db1b7b..3b7cff3c0 100644 --- a/packages/runtime-core/src/provider/graph/cleanup_enrollment.rs +++ b/packages/runtime-core/src/provider/graph/cleanup_enrollment.rs @@ -117,7 +117,8 @@ pub(super) fn ordinary_mutation(root: &Path, receipt: &Receipt) -> Result<(), Ca } pub(super) fn retention(root: &Path, receipt: &Receipt) -> Result<(), CandidateError> { #[cfg(target_os = "macos")] - if super::live_owner_cleanup::retained(root, receipt)? + if super::absent_publication_cleanup::retained_current(root, receipt)? + || super::live_owner_cleanup::retained(root, receipt)? || super::dead_owner_cleanup::retained(root, receipt)? || super::absent_publication_cleanup::retained(root, receipt)? { diff --git a/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs b/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs index d8b89dc68..065371f5e 100644 --- a/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs +++ b/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs @@ -384,6 +384,40 @@ fn finish_retirement( engine.guest().verify() } +/// A committed older cleanup is diagnostic history only after its exact stopped +/// generation and listener retirement are independently confirmed. +pub(super) fn require_historical_recovery( + root: &Path, + current: &Receipt, +) -> Result<(), CandidateError> { + if exists(&root.join("live-owner-cleanup.pending"))? { + return Err(refused()); + } + if !exists(&root.join(FILE))? { + return Ok(()); + } + let prior: Intent = state::read(&root.join(FILE))?; + let complete = prior.complete_sha256.as_deref().ok_or_else(refused)?; + let stopped = + restore_history::completed_for_recovery(root, current, complete)?.ok_or_else(refused)?; + validate(&prior, &stopped, &prior.original_sha256, &prior.boot)?; + if !prior.listeners_retired + || !stopped.resources.iter().any(|(key, old)| { + old.kind == Kind::Container + && old.id.as_ref().is_some_and(|id| { + current + .resources + .get(key) + .and_then(|now| now.id.as_ref()) + .is_some_and(|current_id| current_id != id) + }) + }) + { + return Err(refused()); + } + Ok(()) +} + pub(super) fn retained(root: &Path, receipt: &Receipt) -> Result { if !exists(&root.join(FILE))? { return Ok(false); From 9da7305f62f0b3e3ee6bc1d9379915861e86a818 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 04:45:49 -0400 Subject: [PATCH 14/25] test(runtime): stabilize source recovery lifecycle fixtures --- .../native_test/source_device_rebind.rs | 9 ++- .../lifecycle/host_filesystem/tests.rs | 72 +++++++++++++++---- 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test/source_device_rebind.rs b/packages/runtime-core/src/provider/graph/foreground/native_test/source_device_rebind.rs index a44d6326c..e7d6660ee 100644 --- a/packages/runtime-core/src/provider/graph/foreground/native_test/source_device_rebind.rs +++ b/packages/runtime-core/src/provider/graph/foreground/native_test/source_device_rebind.rs @@ -468,11 +468,6 @@ fn selected_source_device_rebind_restores_same_run_and_two_ordinary_generations( fs::read(graph_root.join("state.json")).unwrap(), original_ready_bytes ); - fs::write(&marker_path, b"source-host-edit\n").unwrap(); - shared_source_marker(&binary, &candidate, &run, b"source-host-edit\n", deadline); - fs::write(&marker_path, initial_marker).unwrap(); - shared_source_marker(&binary, &candidate, &run, initial_marker, deadline); - // The selected witness is now historical. Each later cleanup/restore takes // an ordinary generation and reads the same named retained data. for generation in 0..2 { @@ -516,6 +511,10 @@ fn selected_source_device_rebind_restores_same_run_and_two_ordinary_generations( share.device ); } + // A source edit changes the reviewed plan's metadata identity even if its + // bytes are changed back. Prove the live share after all pinned-plan restores. + fs::write(&marker_path, b"source-host-edit\n").unwrap(); + shared_source_marker(&binary, &candidate, &run, b"source-host-edit\n", deadline); let removed = checked_cli( "remove-selected-data", &binary, diff --git a/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs b/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs index 4c6cc502f..89ebeda4b 100644 --- a/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs +++ b/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs @@ -1,6 +1,39 @@ use super::*; use crate::provider::{NetworkIntent, Profile, ProjectShareIntent}; -use std::{fs::File, os::fd::AsRawFd, path::PathBuf, process::Command}; +use std::{ + fs::{self, File}, + io::ErrorKind, + os::{fd::AsRawFd, unix::fs::DirBuilderExt}, + path::PathBuf, + process::Command, + sync::atomic::{AtomicU64, Ordering}, + time::{SystemTime, UNIX_EPOCH}, +}; + +fn private_pool_directory_at(timestamp_nanos: u128, sequence: &AtomicU64) -> PathBuf { + let base = fs::canonicalize(std::env::temp_dir()).unwrap(); + loop { + let directory = base.join(format!( + "hack-device-recovery-{}-{timestamp_nanos}-{}", + std::process::id(), + sequence.fetch_add(1, Ordering::Relaxed) + )); + match fs::DirBuilder::new().mode(0o700).create(&directory) { + Ok(()) => return directory, + Err(error) if error.kind() == ErrorKind::AlreadyExists => continue, + Err(error) => panic!("cannot create private host-filesystem fixture: {error}"), + } + } +} + +fn private_pool_directory() -> PathBuf { + static NEXT: AtomicU64 = AtomicU64::new(0); + let timestamp_nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + private_pool_directory_at(timestamp_nanos, &NEXT) +} struct Pool { candidate: Candidate, @@ -15,17 +48,7 @@ impl Pool { child.kill().unwrap(); child.wait().unwrap(); let boot = process.start_micros + 1; - let directory = fs::canonicalize(std::env::temp_dir()) - .unwrap() - .join(format!( - "hack-device-recovery-{}-{}", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - state::private_directory(&directory).unwrap(); + let directory = private_pool_directory(); let candidate = Candidate::discover(&directory).unwrap(); let operation = state::Lock::acquire(&root(&candidate)).unwrap(); let project = directory.join("app"); @@ -93,6 +116,31 @@ impl Pool { assert!(self.owner.short_home.symlink_metadata().is_err()); } } + +#[test] +fn same_timestamp_parallel_roots_preserve_an_existing_directory() { + let timestamp_nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let sequence = AtomicU64::new(0); + let (first, second) = std::thread::scope(|scope| { + let first = scope.spawn(|| private_pool_directory_at(timestamp_nanos, &sequence)); + let second = scope.spawn(|| private_pool_directory_at(timestamp_nanos, &sequence)); + (first.join().unwrap(), second.join().unwrap()) + }); + assert_ne!(first, second); + let sentinel = first.join("sentinel"); + fs::write(&sentinel, b"untouched").unwrap(); + let restarted_sequence = AtomicU64::new(0); + let third = private_pool_directory_at(timestamp_nanos, &restarted_sequence); + assert_ne!(third, first); + assert_ne!(third, second); + assert_eq!(fs::read(sentinel).unwrap(), b"untouched"); + for directory in [first, second, third] { + fs::remove_dir_all(directory).unwrap(); + } +} impl Drop for Pool { fn drop(&mut self) { let _ = fs::remove_file(&self.owner.short_home); From dddebc81508e916b2a297e4cf599f51d057ebe7e Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 05:08:44 -0400 Subject: [PATCH 15/25] fix(cli): preserve legacy namespaces during retained branch startup --- docs/guides/native-candidate.md | 11 ++ src/backends/native-project-restore.ts | 41 ++++-- src/backends/native-project-review.ts | 65 +++++++++- src/backends/native-project-start.ts | 1 + tests/native-project-restore.test.ts | 11 ++ tests/native-project-review.test.ts | 170 ++++++++++++++++++++++++- tests/native-project-start.test.ts | 5 + 7 files changed, 289 insertions(+), 15 deletions(-) diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 083e8b35a..f991b2bf1 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -93,6 +93,17 @@ still apply. A same-checkout branch namespace does not create an isolated source tree. These planning contracts do not qualify multiple independent source roots inside one VM pool. +An existing frontend branch mapping may retain a graph admitted before native +branch namespaces. If its current scoped review differs, the frontend first +checks native restore selection against that exact saved run, owner, namespace +and plan. A native unbranched review of the same canonical project and unchanged +original Compose must then return the saved namespace. Only that retained run +continues with unbranched review and execution arguments. Restore selection and +shared-source semantic compatibility are checked again before admission; mappings, +graph receipts and ownership namespaces are not rewritten. Fresh and already +branch-scoped graphs keep their normal branch namespace. This compatibility path +does not authorize adopting an unrelated branch or project. + ## Manual candidate upgrade and rollback Candidate bundles are selected by their full path. There is no automatic candidate diff --git a/src/backends/native-project-restore.ts b/src/backends/native-project-restore.ts index 1f9885d4f..da477f80b 100644 --- a/src/backends/native-project-restore.ts +++ b/src/backends/native-project-restore.ts @@ -9,6 +9,30 @@ import { } from "./native-runtime-client.ts"; const SHA = /^[a-f0-9]{64}$/; +/** Recheck native authority without deriving or translating any ownership identity. */ +export function validateNativeRestoreSelection(opts: { + readonly selected: unknown; + readonly saved: NativeProjectRun; + readonly namespace: string; +}): { readonly generation: string } { + const { selected, saved } = opts; + if ( + !isRecord(selected) || + selected.run !== saved.run || + selected.owner !== saved.owner || + selected.namespace !== opts.namespace || + selected.namespace !== saved.namespace || + selected.plan !== saved.planId || + typeof selected.generation !== "string" || + !SHA.test(selected.generation) + ) { + throw new Error( + "Native restore selection changed; retained data was not adopted." + ); + } + return { generation: selected.generation }; +} + /** Eligible routes consult current native provenance even when reverting to the ownership plan. */ export function nativeReviewNeedsCompatibility(opts: { readonly saved: NativeProjectRun; @@ -94,15 +118,14 @@ export async function selectNativeProjectRestore(opts: { cwd: opts.projectRoot, args: ["graph", "restore-selection", "--run-id", saved.run, "--json"], }); + const selection = validateNativeRestoreSelection({ + selected, + saved, + namespace: opts.review.namespace, + }); if ( - !isRecord(selected) || - selected.run !== saved.run || - selected.owner !== saved.owner || - selected.namespace !== opts.review.namespace || - selected.namespace !== saved.namespace || - selected.plan !== saved.planId || - typeof selected.generation !== "string" || - !SHA.test(selected.generation) + opts.review.retainedGeneration !== undefined && + opts.review.retainedGeneration !== selection.generation ) { throw new Error( "Native restore selection changed; retained data was not adopted." @@ -122,7 +145,7 @@ export async function selectNativeProjectRestore(opts: { : null; return { run: saved.run, - flags: ["--expect-generation", selected.generation], + flags: ["--expect-generation", selection.generation], restoring: true, ...(sourceRevision ? { sourceRevision } : {}), }; diff --git a/src/backends/native-project-review.ts b/src/backends/native-project-review.ts index 384be4738..40a6be8ed 100644 --- a/src/backends/native-project-review.ts +++ b/src/backends/native-project-review.ts @@ -1,9 +1,11 @@ -import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { chmod, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, relative } from "node:path"; import { isRecord } from "../lib/guards.ts"; import { nativeProjectBranchArgs } from "./native-project-branch.ts"; import type { NativeProjectInput } from "./native-project-input.ts"; +import { validateNativeRestoreSelection } from "./native-project-restore.ts"; +import type { NativeProjectRun } from "./native-project-run.ts"; import { invokeNativeRuntime, type NativeRuntimeSelection, @@ -15,6 +17,8 @@ export interface NativeProjectReview { readonly planId: string; readonly namespace: string; readonly report: Readonly>; + /** Exact legacy fallback selection; restore must observe it again after review. */ + readonly retainedGeneration?: string; /** Reuse unchanged for source publication and admission within the callback. */ readonly projectArgs: readonly string[]; } @@ -32,6 +36,8 @@ export async function withNativeProjectReview(opts: { readonly profiles?: readonly string[]; readonly branch?: string | null; readonly input: NativeProjectInput; + readonly retained?: NativeProjectRun; + readonly invoke?: typeof invokeNativeRuntime; readonly run: (review: NativeProjectReview) => Promise; }): Promise { const file = relative(opts.projectRoot, opts.composeFile); @@ -44,7 +50,7 @@ export async function withNativeProjectReview(opts: { "Native project review requires an owned original Compose file." ); } - const projectArgs = [ + let projectArgs = [ "--project", opts.projectRoot, "--file", @@ -52,7 +58,8 @@ export async function withNativeProjectReview(opts: { ...nativeProjectBranchArgs(opts.branch), ...(opts.profiles ?? []).flatMap((profile) => ["--profile", profile]), ]; - const original = await invokeNativeRuntime({ + const invoke = opts.invoke ?? invokeNativeRuntime; + const original = await invoke({ runtime: opts.runtime, cwd: opts.projectRoot, args: ["project", "plan", ...projectArgs, "--json"], @@ -67,7 +74,54 @@ export async function withNativeProjectReview(opts: { "Native project input changed before review; prepare it again." ); } - const namespace = original.plan.namespace; + let namespace = original.plan.namespace; + let retainedGeneration: string | undefined; + if (opts.branch && opts.retained && namespace !== opts.retained.namespace) { + // Legacy mappings may predate native branch namespaces. Only native proofs + // for this exact retained run and canonical project permit unscoped review. + const selected = await invoke({ + runtime: opts.runtime, + cwd: opts.projectRoot, + args: [ + "graph", + "restore-selection", + "--run-id", + opts.retained.run, + "--json", + ], + }); + retainedGeneration = validateNativeRestoreSelection({ + selected, + saved: opts.retained, + namespace: opts.retained.namespace, + }).generation; + const unbranchedArgs = [ + "--project", + opts.projectRoot, + "--file", + file, + ...(opts.profiles ?? []).flatMap((profile) => ["--profile", profile]), + ]; + const unbranched = await invoke({ + runtime: opts.runtime, + cwd: opts.projectRoot, + args: ["project", "plan", ...unbranchedArgs, "--json"], + }); + const canonicalProject = await realpath(opts.projectRoot); + if ( + !(isRecord(unbranched) && isRecord(unbranched.plan)) || + unbranched.plan.namespace !== opts.retained.namespace || + original.plan.source !== canonicalProject || + unbranched.plan.source !== canonicalProject || + unbranched.plan.compose_sha256 !== opts.input.originalSha256 + ) { + throw new Error( + "Native retained project review changed; retained data was not adopted." + ); + } + projectArgs = unbranchedArgs; + namespace = opts.retained.namespace; + } const directory = await mkdtemp(join(tmpdir(), "hack-native-review-")); try { await chmod(directory, 0o700); @@ -85,7 +139,7 @@ export async function withNativeProjectReview(opts: { "--expect-namespace", namespace, ]; - const report = await invokeNativeRuntime({ + const report = await invoke({ runtime: opts.runtime, cwd: opts.projectRoot, args: ["project", "plan", ...normalizedArgs, "--json"], @@ -102,6 +156,7 @@ export async function withNativeProjectReview(opts: { planId: report.plan_id, namespace, report, + ...(retainedGeneration ? { retainedGeneration } : {}), projectArgs: normalizedArgs, }); } finally { diff --git a/src/backends/native-project-start.ts b/src/backends/native-project-start.ts index d900650a1..e286c0180 100644 --- a/src/backends/native-project-start.ts +++ b/src/backends/native-project-start.ts @@ -886,6 +886,7 @@ export async function startNativeProject(opts: { composeFile: opts.composeFile, profiles, branch: opts.scope.branch, + retained: restore, input: pinned, run: async (review) => { requireEnrollmentCompatible(review.report.plan); diff --git a/tests/native-project-restore.test.ts b/tests/native-project-restore.test.ts index acd673a1e..7e539d6ba 100644 --- a/tests/native-project-restore.test.ts +++ b/tests/native-project-restore.test.ts @@ -173,3 +173,14 @@ test("eligible hostname rollback checks current native provenance even at the ow }) ).rejects.toThrow("route change refused"); }); + +test("legacy review refuses a different valid restore generation", async () => { + await expect( + selectNativeProjectRestore({ + ...options, + restore: saved, + review: { ...options.review, retainedGeneration: "1".repeat(64) }, + invoke: async () => ({ ...selected, generation: "2".repeat(64) }), + }) + ).rejects.toThrow("restore selection changed"); +}); diff --git a/tests/native-project-review.test.ts b/tests/native-project-review.test.ts index e47f7b814..dcc715ba6 100644 --- a/tests/native-project-review.test.ts +++ b/tests/native-project-review.test.ts @@ -1,9 +1,11 @@ import { afterEach, expect, test } from "bun:test"; -import { chmod, mkdtemp, readFile, rm, stat } from "node:fs/promises"; +import { chmod, mkdtemp, readFile, realpath, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { prepareNativeProjectInput } from "../src/backends/native-project-input.ts"; +import { selectNativeProjectRestore } from "../src/backends/native-project-restore.ts"; import { withNativeProjectReview } from "../src/backends/native-project-review.ts"; +import type { invokeNativeRuntime } from "../src/backends/native-runtime-client.ts"; const roots: string[] = []; afterEach(async () => { @@ -131,3 +133,169 @@ test("noncanonical branch review refuses before invoking the executor", async () }) ).rejects.toThrow("canonical branch"); }); + +const retained = { + run: "d".repeat(32), + owner: "e".repeat(32), + namespace: "a".repeat(64), + planId: "b".repeat(64), +}; + +async function legacyFixture() { + const options = await fixture(); + const calls: string[][] = []; + const selected: Record = { + run: retained.run, + owner: retained.owner, + namespace: retained.namespace, + plan: retained.planId, + generation: "f".repeat(64), + }; + const unbranched: Record = { + namespace: retained.namespace, + source: await realpath(options.projectRoot), + compose_sha256: options.input.originalSha256, + }; + const invoke: typeof invokeNativeRuntime = async ({ args }) => { + calls.push([...args]); + if (args[0] === "graph") { + return selected; + } + if (!args.includes("--branch")) { + return { plan_id: retained.planId, plan: unbranched }; + } + return { + plan_id: retained.planId, + plan: { ...unbranched, namespace: "c".repeat(64) }, + }; + }; + return { options, calls, selected, unbranched, invoke }; +} + +test("legacy retained review uses native unbranched authority and rechecks restore selection", async () => { + const { options, calls, invoke } = await legacyFixture(); + let temporary = ""; + await withNativeProjectReview({ + ...options, + branch: "feature-a", + retained, + invoke, + run: async (review) => { + expect(review.namespace).toBe(retained.namespace); + expect(review.projectArgs).not.toContain("--branch"); + temporary = + review.projectArgs[ + review.projectArgs.indexOf("--normalized-file") + 1 + ] ?? ""; + expect(await Bun.file(temporary).text()).toBe( + options.input.normalizedComposeJson + ); + const selection = await selectNativeProjectRestore({ + runtime: options.runtime, + projectRoot: options.projectRoot, + restore: retained, + review, + invoke, + }); + expect(selection.run).toBe(retained.run); + expect(selection.flags).toEqual(["--expect-generation", "f".repeat(64)]); + }, + }); + expect(calls.map((args) => args.slice(0, 2).join(" "))).toEqual([ + "project plan", + "graph restore-selection", + "project plan", + "project plan", + "graph restore-selection", + ]); + expect(await Bun.file(temporary).exists()).toBe(false); +}); + +test("fresh and current branch-native review never consult legacy restore authority", async () => { + for (const saved of [undefined, { ...retained, namespace: "c".repeat(64) }]) { + const { options, calls, invoke } = await legacyFixture(); + await withNativeProjectReview({ + ...options, + branch: "feature-a", + retained: saved, + invoke, + run: async (review) => { + expect(review.namespace).toBe("c".repeat(64)); + expect(review.projectArgs).toContain("--branch"); + }, + }); + expect(calls).toHaveLength(2); + expect(calls.every((args) => args.includes("--branch"))).toBe(true); + } +}); + +test("legacy review rejects changed native selection before unbranched review", async () => { + for (const key of ["run", "owner", "namespace", "plan", "generation"]) { + const { options, calls, selected, invoke } = await legacyFixture(); + selected[key] = "invalid"; + await expect( + withNativeProjectReview({ + ...options, + branch: "feature-a", + retained, + invoke, + run: async () => { + throw new Error("unexpected callback"); + }, + }) + ).rejects.toThrow("restore selection changed"); + expect(calls).toHaveLength(2); + } +}); + +test("legacy review rejects a different project, namespace or edited original without admission", async () => { + for (const key of ["source", "namespace", "compose_sha256"]) { + const { options, calls, unbranched, invoke } = await legacyFixture(); + const originalInvoke: typeof invokeNativeRuntime = async (request) => { + if (request.args[0] === "project" && !request.args.includes("--branch")) { + unbranched[key] = "foreign"; + } + return await invoke(request); + }; + await expect( + withNativeProjectReview({ + ...options, + branch: "feature-a", + retained, + invoke: originalInvoke, + run: async () => { + throw new Error("unexpected callback"); + }, + }) + ).rejects.toThrow("retained project review changed"); + expect(calls).toHaveLength(3); + } +}); + +test("legacy review cleans temporary input and refuses selection drift after review", async () => { + const { options, selected, invoke } = await legacyFixture(); + let temporary = ""; + await expect( + withNativeProjectReview({ + ...options, + branch: "feature-a", + retained, + invoke, + run: async (review) => { + temporary = + review.projectArgs[ + review.projectArgs.indexOf("--normalized-file") + 1 + ] ?? ""; + selected.owner = "0".repeat(32); + await selectNativeProjectRestore({ + runtime: options.runtime, + projectRoot: options.projectRoot, + restore: retained, + review, + invoke, + }); + }, + }) + ).rejects.toThrow("restore selection changed"); + expect(await Bun.file(temporary).exists()).toBe(false); +}); diff --git a/tests/native-project-start.test.ts b/tests/native-project-start.test.ts index 4143ed1da..ced949553 100644 --- a/tests/native-project-start.test.ts +++ b/tests/native-project-start.test.ts @@ -1452,6 +1452,11 @@ test("restore startup passes the retained run and selected generation to its own namespace: "b".repeat(64), planId: "a".repeat(64), }; + const review = opts.dependencies.review!; + opts.dependencies.review = async (request) => { + expect(request.retained).toEqual(saved); + return await review(request); + }; const invoke = opts.dependencies.invoke!; const serve = opts.dependencies.serve!; opts.dependencies.invoke = async (request) => From 46fe63e1d0510ae1c2b0b299a9f32c2c10aee440 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 05:38:47 -0400 Subject: [PATCH 16/25] fix(cli): select retained routing before branch normalization --- docs/guides/native-candidate.md | 12 +- .../native-project-restart-preflight.ts | 23 +++- src/backends/native-project-review.ts | 119 +++++++++++++++-- src/backends/native-project-start.ts | 47 ++++++- tests/native-project-review.test.ts | 121 +++++++++++++++++- tests/native-project-start.test.ts | 100 ++++++++++++++- 6 files changed, 393 insertions(+), 29 deletions(-) diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index f991b2bf1..131f4db46 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -98,10 +98,14 @@ branch namespaces. If its current scoped review differs, the frontend first checks native restore selection against that exact saved run, owner, namespace and plan. A native unbranched review of the same canonical project and unchanged original Compose must then return the saved namespace. Only that retained run -continues with unbranched review and execution arguments. Restore selection and -shared-source semantic compatibility are checked again before admission; mappings, -graph receipts and ownership namespaces are not rewritten. Fresh and already -branch-scoped graphs keep their normal branch namespace. This compatibility path +continues with unbranched review and execution arguments and keeps its original +adapted route labels, before branch hostname rewriting. This preserves legacy +source contracts that did not enroll hostname changes. The selected generation, +restore selection and shared-source semantic compatibility are checked again +before admission; mappings, graph receipts and ownership namespaces are not +rewritten. An active legacy graph still refuses restart preflight before cleanup; +this stopped-run compatibility path does not migrate its live identity. Fresh and +already branch-scoped graphs keep their normal branch namespace. This compatibility path does not authorize adopting an unrelated branch or project. ## Manual candidate upgrade and rollback diff --git a/src/backends/native-project-restart-preflight.ts b/src/backends/native-project-restart-preflight.ts index eee54851b..b277def9b 100644 --- a/src/backends/native-project-restart-preflight.ts +++ b/src/backends/native-project-restart-preflight.ts @@ -5,7 +5,6 @@ import { dirname, join } from "node:path"; import { isRecord } from "../lib/guards.ts"; import { adaptNativeAwsEnvironment } from "./native-aws-environment.ts"; import { prepareNativeProjectAdaptation } from "./native-project-adaptation.ts"; -import { prepareNativeProjectBranch } from "./native-project-branch.ts"; import { discoverNativeHostDependency, type NativeHostDependency, @@ -21,7 +20,11 @@ import { verifyNativeSourceCompatibility, } from "./native-project-restore.ts"; import { confirmedNativeRetainedGraph } from "./native-project-retained.ts"; -import { withNativeProjectReview } from "./native-project-review.ts"; +import { + prepareNativeReviewBranch, + selectNativeProjectReviewIdentity, + withNativeProjectReview, +} from "./native-project-review.ts"; import { type NativeProjectRun, type NativeProjectRunScope, @@ -41,6 +44,7 @@ const DEFAULTS = { dependencies: readNativeHostDependencies, invoke: invokeNativeRuntime, review: withNativeProjectReview, + selectReview: selectNativeProjectReviewIdentity, }; const IMAGE = /^sha256:[a-f0-9]{64}$/; const SHA = /^[a-f0-9]{64}$/; @@ -268,11 +272,19 @@ export async function preflightNativeRestart(opts: { input, path: opts.adaptationFile, }); - input = await prepareNativeProjectBranch({ - input, + const reviewedBranch = await prepareNativeReviewBranch({ + runtime: opts.runtime, scope: opts.scope, composeFile: opts.composeFile, + profiles: selected.profiles, + input, + retained: opts.run, + retainedMode: opts.cleanedRetry ? "stopped" : "active", + invoke: deps.invoke, + select: deps.selectReview, }); + input = reviewedBranch.input; + const reviewIdentity = reviewedBranch.identity; if (selected.aws) { input = (await deps.aws({ input, ...selected.aws })).input; } @@ -326,6 +338,9 @@ export async function preflightNativeRestart(opts: { composeFile: opts.composeFile, profiles: selected.profiles, branch: opts.scope.branch, + retained: opts.run, + retainedMode: opts.cleanedRetry ? "stopped" : "active", + reviewIdentity, input: { ...input, normalizedComposeJson: JSON.stringify(compose) }, run: async (review) => { if (review.namespace !== opts.run.namespace) { diff --git a/src/backends/native-project-review.ts b/src/backends/native-project-review.ts index 40a6be8ed..088fa8e8c 100644 --- a/src/backends/native-project-review.ts +++ b/src/backends/native-project-review.ts @@ -2,10 +2,16 @@ import { chmod, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, relative } from "node:path"; import { isRecord } from "../lib/guards.ts"; -import { nativeProjectBranchArgs } from "./native-project-branch.ts"; +import { + nativeProjectBranchArgs, + prepareNativeProjectBranch, +} from "./native-project-branch.ts"; import type { NativeProjectInput } from "./native-project-input.ts"; import { validateNativeRestoreSelection } from "./native-project-restore.ts"; -import type { NativeProjectRun } from "./native-project-run.ts"; +import type { + NativeProjectRun, + NativeProjectRunScope, +} from "./native-project-run.ts"; import { invokeNativeRuntime, type NativeRuntimeSelection, @@ -23,13 +29,15 @@ export interface NativeProjectReview { readonly projectArgs: readonly string[]; } -/** - * Native code owns namespace and plan identities. Hold only public normalized - * bytes in a private temporary directory while the caller publishes/starts the - * exact plan. The native side rechecks original input identity at every effect. - * Managed values remain in the caller's in-memory input, outside this file. - */ -export async function withNativeProjectReview(opts: { +/** Original-input identity chosen before any branch route normalization. */ +export interface NativeProjectReviewIdentity { + readonly namespace: string; + readonly branch: string | null; + readonly projectArgs: readonly string[]; + readonly retainedGeneration?: string; +} + +export async function selectNativeProjectReviewIdentity(opts: { readonly runtime: NativeRuntimeSelection; readonly projectRoot: string; readonly composeFile: string; @@ -38,8 +46,8 @@ export async function withNativeProjectReview(opts: { readonly input: NativeProjectInput; readonly retained?: NativeProjectRun; readonly invoke?: typeof invokeNativeRuntime; - readonly run: (review: NativeProjectReview) => Promise; -}): Promise { + readonly retainedMode?: "stopped" | "active"; +}): Promise { const file = relative(opts.projectRoot, opts.composeFile); if ( !file || @@ -77,6 +85,11 @@ export async function withNativeProjectReview(opts: { let namespace = original.plan.namespace; let retainedGeneration: string | undefined; if (opts.branch && opts.retained && namespace !== opts.retained.namespace) { + if (opts.retainedMode === "active") { + throw new Error( + "Active legacy native branch review requires explicit retained recovery; the current graph was not stopped." + ); + } // Legacy mappings may predate native branch namespaces. Only native proofs // for this exact retained run and canonical project permit unscoped review. const selected = await invoke({ @@ -122,6 +135,44 @@ export async function withNativeProjectReview(opts: { projectArgs = unbranchedArgs; namespace = opts.retained.namespace; } + return { + namespace, + branch: retainedGeneration ? null : (opts.branch ?? null), + projectArgs, + ...(retainedGeneration ? { retainedGeneration } : {}), + }; +} + +/** + * Native code owns namespace and plan identities. Hold only public normalized + * bytes in a private temporary directory while the caller publishes/starts the + * exact plan. The native side rechecks original input identity at every effect. + * Managed values remain in the caller's in-memory input, outside this file. + */ +export async function withNativeProjectReview(opts: { + readonly runtime: NativeRuntimeSelection; + readonly projectRoot: string; + readonly composeFile: string; + readonly profiles?: readonly string[]; + readonly branch?: string | null; + readonly input: NativeProjectInput; + readonly retained?: NativeProjectRun; + readonly invoke?: typeof invokeNativeRuntime; + readonly retainedMode?: "stopped" | "active"; + readonly reviewIdentity?: NativeProjectReviewIdentity; + readonly run: (review: NativeProjectReview) => Promise; +}): Promise { + const identity = await selectNativeProjectReviewIdentity(opts); + if ( + opts.reviewIdentity && + JSON.stringify(opts.reviewIdentity) !== JSON.stringify(identity) + ) { + throw new Error( + "Native project review selection changed; retained data was not adopted." + ); + } + const { namespace, projectArgs, retainedGeneration } = identity; + const invoke = opts.invoke ?? invokeNativeRuntime; const directory = await mkdtemp(join(tmpdir(), "hack-native-review-")); try { await chmod(directory, 0o700); @@ -163,3 +214,49 @@ export async function withNativeProjectReview(opts: { await rm(directory, { recursive: true, force: true }); } } + +/** Choose retained native identity before rewriting any adapted route labels. */ +export async function prepareNativeReviewBranch(opts: { + readonly runtime: NativeRuntimeSelection; + readonly phase?: "before-runtime" | "after-runtime"; + readonly scope: NativeProjectRunScope; + readonly composeFile: string; + readonly profiles?: readonly string[]; + readonly input: NativeProjectInput; + readonly retained?: NativeProjectRun; + readonly retainedMode?: "stopped" | "active"; + readonly invoke?: typeof invokeNativeRuntime; + readonly select?: typeof selectNativeProjectReviewIdentity; +}): Promise<{ + input: NativeProjectInput; + identity?: NativeProjectReviewIdentity; +}> { + nativeProjectBranchArgs(opts.scope.branch); + const deferred = Boolean(opts.retained && opts.scope.branch); + if ( + (opts.phase === "before-runtime" && deferred) || + (opts.phase === "after-runtime" && !deferred) + ) { + return { input: opts.input }; + } + const identity = + opts.retained && opts.scope.branch + ? await (opts.select ?? selectNativeProjectReviewIdentity)({ + runtime: opts.runtime, + projectRoot: opts.scope.projectRoot, + composeFile: opts.composeFile, + profiles: opts.profiles, + branch: opts.scope.branch, + input: opts.input, + retained: opts.retained, + retainedMode: opts.retainedMode, + invoke: opts.invoke, + }) + : undefined; + const input = await prepareNativeProjectBranch({ + input: opts.input, + composeFile: opts.composeFile, + scope: identity ? { ...opts.scope, branch: identity.branch } : opts.scope, + }); + return { input, ...(identity ? { identity } : {}) }; +} diff --git a/src/backends/native-project-start.ts b/src/backends/native-project-start.ts index e286c0180..3f0ec4838 100644 --- a/src/backends/native-project-start.ts +++ b/src/backends/native-project-start.ts @@ -14,7 +14,6 @@ import { preparedBaseArguments, } from "./native-prepared-base.ts"; import { prepareNativeProjectAdaptation } from "./native-project-adaptation.ts"; -import { prepareNativeProjectBranch } from "./native-project-branch.ts"; import { nativeSharedSourceFlags, nativeStartCacheSource, @@ -40,7 +39,11 @@ import { verifyNativeResumedRetainedGraph, verifyNativeRetainedMapping, } from "./native-project-retained-startup.ts"; -import { withNativeProjectReview } from "./native-project-review.ts"; +import { + prepareNativeReviewBranch, + selectNativeProjectReviewIdentity, + withNativeProjectReview, +} from "./native-project-review.ts"; import { hasOnlyNativeSupportedLabels, nativeBridgeCapacity, @@ -211,6 +214,7 @@ type Dependencies = { prepareStorage: typeof prepareNativeProjectRunStorage; adaptAws: typeof adaptNativeAwsEnvironment; review: typeof withNativeProjectReview; + selectReview: typeof selectNativeProjectReviewIdentity; serve: typeof serveNativeProjectGraph; https: typeof acquireNativeHttpsLease; recoverHttps: typeof recoverNativeHttpsLease; @@ -226,6 +230,7 @@ const DEFAULTS: Dependencies = { prepareStorage: prepareNativeProjectRunStorage, adaptAws: adaptNativeAwsEnvironment, review: withNativeProjectReview, + selectReview: selectNativeProjectReviewIdentity, serve: serveNativeProjectGraph, https: acquireNativeHttpsLease, recoverHttps: recoverNativeHttpsLease, @@ -735,11 +740,16 @@ export async function startNativeProject(opts: { input, path: opts.adaptationFile, }); - input = await prepareNativeProjectBranch({ - input, - scope: opts.scope, - composeFile: opts.composeFile, - }); + input = ( + await prepareNativeReviewBranch({ + runtime: opts.runtime, + scope: opts.scope, + composeFile: opts.composeFile, + input, + retained: restore, + phase: "before-runtime", + }) + ).input; let specs = prepareNativeProjectServices( input, opts.dependencyFile !== undefined @@ -854,6 +864,28 @@ export async function startNativeProject(opts: { run: retained, invoke: deps.invoke, }); + const reviewedBranch = await prepareNativeReviewBranch({ + runtime: opts.runtime, + scope: opts.scope, + composeFile: opts.composeFile, + profiles, + input, + retained: restore, + invoke: deps.invoke, + select: deps.selectReview, + phase: "after-runtime", + }); + input = reviewedBranch.input; + const reviewIdentity = reviewedBranch.identity; + requireActiveStartup(controller.signal); + specs = prepareNativeProjectServices( + input, + opts.dependencyFile !== undefined + ); + prepareNativeDependencyServices({ + dependencies: hostDependencies, + services: specs, + }); for (const spec of Object.values(specs)) { if (controller.signal.aborted) { throw refused(); @@ -887,6 +919,7 @@ export async function startNativeProject(opts: { profiles, branch: opts.scope.branch, retained: restore, + reviewIdentity, input: pinned, run: async (review) => { requireEnrollmentCompatible(review.report.plan); diff --git a/tests/native-project-review.test.ts b/tests/native-project-review.test.ts index dcc715ba6..1a3de8aa8 100644 --- a/tests/native-project-review.test.ts +++ b/tests/native-project-review.test.ts @@ -4,7 +4,11 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { prepareNativeProjectInput } from "../src/backends/native-project-input.ts"; import { selectNativeProjectRestore } from "../src/backends/native-project-restore.ts"; -import { withNativeProjectReview } from "../src/backends/native-project-review.ts"; +import { + prepareNativeReviewBranch, + selectNativeProjectReviewIdentity, + withNativeProjectReview, +} from "../src/backends/native-project-review.ts"; import type { invokeNativeRuntime } from "../src/backends/native-runtime-client.ts"; const roots: string[] = []; @@ -299,3 +303,118 @@ test("legacy review cleans temporary input and refuses selection drift after rev ).rejects.toThrow("restore selection changed"); expect(await Bun.file(temporary).exists()).toBe(false); }); + +test("legacy identity is selected before route normalization and original adapted labels survive", async () => { + const { options, invoke } = await legacyFixture(); + const compose = JSON.parse(options.input.normalizedComposeJson); + compose.services.web.labels = { + caddy: "app.hack, app.hack.gy", + "caddy.reverse_proxy": "{{upstreams 3000}}", + }; + const input = { + ...options.input, + normalizedComposeJson: JSON.stringify(compose), + }; + const scope = { + projectRoot: options.projectRoot, + projectDir: join(options.projectRoot, ".hack"), + nativeHome: options.runtime.home, + branch: "feature-a", + }; + const prepared = await prepareNativeReviewBranch({ + ...options, + scope, + input, + retained, + invoke, + }); + expect(prepared.input.normalizedComposeJson).toBe( + input.normalizedComposeJson + ); + expect(prepared.identity?.branch).toBeNull(); + await withNativeProjectReview({ + ...options, + input: prepared.input, + branch: scope.branch, + retained, + invoke, + reviewIdentity: prepared.identity, + run: async (review) => { + expect(review.namespace).toBe(retained.namespace); + expect(review.projectArgs).not.toContain("--branch"); + }, + }); +}); + +test("early identity proof is rechecked after route preparation before normalized publication", async () => { + for (const key of ["generation", "owner"]) { + const { options, selected, calls, invoke } = await legacyFixture(); + const identity = await selectNativeProjectReviewIdentity({ + ...options, + branch: "feature-a", + retained, + invoke, + }); + selected[key] = "1".repeat(key === "generation" ? 64 : 32); + calls.splice(0); + await expect( + withNativeProjectReview({ + ...options, + branch: "feature-a", + retained, + invoke, + reviewIdentity: identity, + run: async () => { + throw new Error("unexpected admission"); + }, + }) + ).rejects.toThrow("selection changed"); + expect(calls.some((args) => args.includes("--normalized-file"))).toBe( + false + ); + } +}); + +test("active branch-native review needs no stopped selection; active legacy mismatch refuses before cleanup", async () => { + const { options, calls, invoke } = await legacyFixture(); + const identity = await selectNativeProjectReviewIdentity({ + ...options, + branch: "feature-a", + retained: { ...retained, namespace: "c".repeat(64) }, + retainedMode: "active", + invoke, + }); + expect(identity.branch).toBe("feature-a"); + expect(calls).toHaveLength(1); + calls.splice(0); + await expect( + selectNativeProjectReviewIdentity({ + ...options, + branch: "feature-a", + retained, + retainedMode: "active", + invoke, + }) + ).rejects.toThrow("Active legacy native branch review"); + expect(calls).toHaveLength(1); + expect(calls[0]?.[0]).toBe("project"); +}); + +test("deferred retained review rejects malformed branch before hooks or runtime selection", async () => { + const { options, calls, invoke } = await legacyFixture(); + await expect( + prepareNativeReviewBranch({ + ...options, + retained, + invoke, + phase: "before-runtime", + scope: { + projectRoot: options.projectRoot, + projectDir: join(options.projectRoot, ".hack"), + nativeHome: options.runtime.home, + branch: "foreign/branch", + }, + }) + ).rejects.toThrow("canonical branch"); + expect(calls).toHaveLength(0); +}); diff --git a/tests/native-project-start.test.ts b/tests/native-project-start.test.ts index ced949553..542d7cdaf 100644 --- a/tests/native-project-start.test.ts +++ b/tests/native-project-start.test.ts @@ -1,5 +1,12 @@ import { afterEach, expect, test } from "bun:test"; -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rm, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { acquireNativeHttpsLease } from "../src/backends/native-https-owner.ts"; @@ -1909,13 +1916,21 @@ test("startup and restart review use identical branch routes after adaptation", }; return { ...input, normalizedComposeJson: JSON.stringify(compose) }; }; + const before = async (input: NativeProjectInput) => { + expect( + JSON.parse(input.normalizedComposeJson).services.web.labels.caddy + ).toBe("feature-a.app.hack.local, feature-a.app.hack.gy"); + return await opts.before(); + }; const observed: NativeProjectInput[] = []; opts.dependencies.review = async (request) => { expect(request.branch).toBe("feature-a"); observed.push(request.input); return await review(request); }; - expect(await startNativeProject({ ...opts, scope, adaptationFile })).toBe(0); + expect( + await startNativeProject({ ...opts, scope, adaptationFile, before }) + ).toBe(0); await preflightNativeRestart({ runtime: opts.runtime, scope, @@ -1935,6 +1950,12 @@ test("startup and restart review use identical branch routes after adaptation", review: opts.dependencies.review, dependencies: async () => [], invoke: async ({ args }) => { + if (args[0] === "project") { + expect(args).toContain("--branch"); + return { + plan: { namespace: "b".repeat(64), compose_sha256: "d".repeat(64) }, + }; + } if (args[1] === "status") { return { network: "internet" }; } @@ -1954,3 +1975,78 @@ test("startup and restart review use identical branch routes after adaptation", .caddy ).toBe("feature-a.app.hack.local, feature-a.app.hack.gy"); }); + +test("retained startup selects legacy labels only after runtime admission and before review", async () => { + for (const changed of [false, true]) { + const { opts } = await fixture(false); + const saved = { + run: "1".repeat(32), + owner: "c".repeat(32), + namespace: "b".repeat(64), + planId: "a".repeat(64), + }; + const prepare = opts.dependencies.prepare!; + opts.dependencies.prepare = async (request) => { + const input = await prepare(request); + const compose = JSON.parse(input.normalizedComposeJson); + compose.services.web.labels = { + caddy: "app.hack.local", + "caddy.tls": "internal", + "caddy.reverse_proxy": "{{upstreams 3000}}", + }; + return { ...input, normalizedComposeJson: JSON.stringify(compose) }; + }; + const invoke = opts.dependencies.invoke!; + let admitted = false; + let reviewed = false; + opts.dependencies.invoke = async (request) => { + if (request.args[0] === "runtime" && request.args[1] === "up") { + admitted = true; + } + if (request.args[0] === "project") { + expect(admitted).toBe(true); + return { + plan: { + namespace: request.args.includes("--branch") + ? "d".repeat(64) + : saved.namespace, + compose_sha256: "d".repeat(64), + source: await realpath(opts.scope.projectRoot), + }, + }; + } + if (request.args[1] === "restore-selection") { + expect(admitted).toBe(true); + return { + ...saved, + owner: changed ? "0".repeat(32) : saved.owner, + plan: saved.planId, + generation: "2".repeat(64), + }; + } + return await invoke(request); + }; + const review = opts.dependencies.review!; + opts.dependencies.review = async (request) => { + reviewed = true; + expect(request.reviewIdentity?.branch).toBeNull(); + expect( + JSON.parse(request.input.normalizedComposeJson).services.web.labels + .caddy + ).toBe("app.hack.local"); + return await review(request); + }; + const running = startNativeProject({ + ...opts, + scope: { ...opts.scope, branch: "feature-a" }, + restore: saved, + }); + if (changed) { + await expect(running).rejects.toThrow("restore selection changed"); + expect(reviewed).toBe(false); + } else { + expect(await running).toBe(0); + expect(reviewed).toBe(true); + } + } +}); From 87f12390180400ef37ac2d6ea5d72e9268a9e12c Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 06:22:00 -0400 Subject: [PATCH 17/25] fix(cli): verify active legacy branch restart identity --- docs/guides/native-candidate.md | 16 +- .../native-project-restart-preflight.ts | 9 + src/backends/native-project-review.ts | 161 +++++++++++++++--- tests/native-project-restart.test.ts | 129 ++++++++++++++ tests/native-project-review.test.ts | 122 +++++++++++-- 5 files changed, 395 insertions(+), 42 deletions(-) diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 131f4db46..4211e6179 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -95,18 +95,22 @@ inside one VM pool. An existing frontend branch mapping may retain a graph admitted before native branch namespaces. If its current scoped review differs, the frontend first -checks native restore selection against that exact saved run, owner, namespace -and plan. A native unbranched review of the same canonical project and unchanged +checks native selection against that exact saved run, owner, namespace and plan: +restore selection for a stopped graph, or authenticated service selection for an +active graph. A native unbranched review of the same canonical project and unchanged original Compose must then return the saved namespace. Only that retained run continues with unbranched review and execution arguments and keeps its original adapted route labels, before branch hostname rewriting. This preserves legacy source contracts that did not enroll hostname changes. The selected generation, restore selection and shared-source semantic compatibility are checked again before admission; mappings, graph receipts and ownership namespaces are not -rewritten. An active legacy graph still refuses restart preflight before cleanup; -this stopped-run compatibility path does not migrate its live identity. Fresh and -already branch-scoped graphs keep their normal branch namespace. This compatibility path -does not authorize adopting an unrelated branch or project. +rewritten. Active legacy restart also rechecks its service, container, boot and +receipt generation after compatibility review, before cleanup eligibility. A +changed selection refuses before stopping the graph. This preflight check does +not make cleanup atomic with that generation; existing ownership and frontend +finalization checks still apply. Fresh and already branch-scoped graphs keep +their normal branch namespace. This compatibility path does not authorize +adopting an unrelated branch or project. ## Manual candidate upgrade and rollback diff --git a/src/backends/native-project-restart-preflight.ts b/src/backends/native-project-restart-preflight.ts index b277def9b..72df9d365 100644 --- a/src/backends/native-project-restart-preflight.ts +++ b/src/backends/native-project-restart-preflight.ts @@ -23,6 +23,7 @@ import { confirmedNativeRetainedGraph } from "./native-project-retained.ts"; import { prepareNativeReviewBranch, selectNativeProjectReviewIdentity, + verifyNativeActiveReview, withNativeProjectReview, } from "./native-project-review.ts"; import { @@ -341,6 +342,7 @@ export async function preflightNativeRestart(opts: { retained: opts.run, retainedMode: opts.cleanedRetry ? "stopped" : "active", reviewIdentity, + invoke: deps.invoke, input: { ...input, normalizedComposeJson: JSON.stringify(compose) }, run: async (review) => { if (review.namespace !== opts.run.namespace) { @@ -365,6 +367,13 @@ export async function preflightNativeRestart(opts: { invoke: deps.invoke, cleanedRetry: opts.cleanedRetry === true, }); + await verifyNativeActiveReview({ + runtime: opts.runtime, + projectRoot: opts.scope.projectRoot, + retained: opts.run, + proof: reviewIdentity?.activeProof, + invoke: deps.invoke, + }); }, }); } diff --git a/src/backends/native-project-review.ts b/src/backends/native-project-review.ts index 088fa8e8c..89ae4ef2c 100644 --- a/src/backends/native-project-review.ts +++ b/src/backends/native-project-review.ts @@ -18,6 +18,8 @@ import { } from "./native-runtime-client.ts"; const DIGEST = /^[a-f0-9]{64}$/; +const CONTROL = /[\x00-\x1f\x7f]/; +const SERVICE = /^[a-zA-Z0-9_.-]{1,128}$/; export interface NativeProjectReview { readonly planId: string; @@ -29,12 +31,120 @@ export interface NativeProjectReview { readonly projectArgs: readonly string[]; } +/** Authenticated active service selection; never a stopped restore generation. */ +export interface NativeActiveReviewProof { + readonly run: string; + readonly owner: string; + readonly namespace: string; + readonly plan: string; + readonly service: string; + readonly container: string; + readonly boot: string; + readonly generation: string; +} + +async function selectActiveReview(opts: { + readonly runtime: NativeRuntimeSelection; + readonly projectRoot: string; + readonly retained: NativeProjectRun; + readonly service: string; + readonly invoke: typeof invokeNativeRuntime; +}): Promise { + const selected = await opts.invoke({ + runtime: opts.runtime, + cwd: opts.projectRoot, + args: [ + "graph", + "run-selection", + "--run-id", + opts.retained.run, + "--service", + opts.service, + "--json", + ], + }); + const { generation } = validateNativeRestoreSelection({ + selected, + saved: opts.retained, + namespace: opts.retained.namespace, + }); + if ( + !isRecord(selected) || + selected.ok !== true || + selected.service !== opts.service || + typeof selected.container !== "string" || + !DIGEST.test(selected.container) || + typeof selected.boot !== "string" || + selected.boot.length < 1 || + selected.boot.length > 128 || + CONTROL.test(selected.boot) + ) { + throw new Error( + "Native active review identity changed; the current graph was not stopped." + ); + } + return { + run: opts.retained.run, + owner: opts.retained.owner, + namespace: opts.retained.namespace, + plan: opts.retained.planId, + service: opts.service, + container: selected.container, + boot: selected.boot, + generation, + }; +} + +function activeReviewService(plan: Readonly>): string { + // `active` means profile-enabled. Native run-selection also verifies admitted + // completed initializers; selecting their identity never executes them again. + const services = isRecord(plan.services) ? plan.services : {}; + const service = Object.keys(services) + .sort() + .find( + (key) => + SERVICE.test(key) && + isRecord(services[key]) && + services[key].active === true + ); + if (!service) { + throw new Error( + "Native active review has no supported service authority; the current graph was not stopped." + ); + } + return service; +} + +/** Repeat the exact active selection after compatibility review, before cleanup eligibility. */ +export async function verifyNativeActiveReview(opts: { + readonly runtime: NativeRuntimeSelection; + readonly projectRoot: string; + readonly retained: NativeProjectRun; + readonly proof?: NativeActiveReviewProof; + readonly invoke?: typeof invokeNativeRuntime; +}): Promise { + if (!opts.proof) { + return; + } + const current = await selectActiveReview({ + ...opts, + service: opts.proof.service, + invoke: opts.invoke ?? invokeNativeRuntime, + }); + if (JSON.stringify(current) !== JSON.stringify(opts.proof)) { + throw new Error( + "Native active review identity changed; the current graph was not stopped." + ); + } +} + /** Original-input identity chosen before any branch route normalization. */ export interface NativeProjectReviewIdentity { readonly namespace: string; readonly branch: string | null; readonly projectArgs: readonly string[]; readonly retainedGeneration?: string; + readonly activeProof?: NativeActiveReviewProof; } export async function selectNativeProjectReviewIdentity(opts: { @@ -83,31 +193,36 @@ export async function selectNativeProjectReviewIdentity(opts: { ); } let namespace = original.plan.namespace; + let branch = opts.branch ?? null; let retainedGeneration: string | undefined; + let activeProof: NativeActiveReviewProof | undefined; if (opts.branch && opts.retained && namespace !== opts.retained.namespace) { if (opts.retainedMode === "active") { - throw new Error( - "Active legacy native branch review requires explicit retained recovery; the current graph was not stopped." - ); + activeProof = await selectActiveReview({ + runtime: opts.runtime, + projectRoot: opts.projectRoot, + retained: opts.retained, + service: activeReviewService(original.plan), + invoke, + }); + } else { + const selected = await invoke({ + runtime: opts.runtime, + cwd: opts.projectRoot, + args: [ + "graph", + "restore-selection", + "--run-id", + opts.retained.run, + "--json", + ], + }); + retainedGeneration = validateNativeRestoreSelection({ + selected, + saved: opts.retained, + namespace: opts.retained.namespace, + }).generation; } - // Legacy mappings may predate native branch namespaces. Only native proofs - // for this exact retained run and canonical project permit unscoped review. - const selected = await invoke({ - runtime: opts.runtime, - cwd: opts.projectRoot, - args: [ - "graph", - "restore-selection", - "--run-id", - opts.retained.run, - "--json", - ], - }); - retainedGeneration = validateNativeRestoreSelection({ - selected, - saved: opts.retained, - namespace: opts.retained.namespace, - }).generation; const unbranchedArgs = [ "--project", opts.projectRoot, @@ -134,12 +249,14 @@ export async function selectNativeProjectReviewIdentity(opts: { } projectArgs = unbranchedArgs; namespace = opts.retained.namespace; + branch = null; } return { namespace, - branch: retainedGeneration ? null : (opts.branch ?? null), + branch, projectArgs, ...(retainedGeneration ? { retainedGeneration } : {}), + ...(activeProof ? { activeProof } : {}), }; } diff --git a/tests/native-project-restart.test.ts b/tests/native-project-restart.test.ts index c7f12d496..03bdd6ce5 100644 --- a/tests/native-project-restart.test.ts +++ b/tests/native-project-restart.test.ts @@ -915,3 +915,132 @@ test("malformed dependency intent refuses cleaned retry without capture or start await rm(listener.directory, { recursive: true, force: true }); } }); + +test("active legacy restart preserves adapted labels and rechecks authenticated selection before cleanup", async () => { + const root = await mkdtemp(join(tmpdir(), "native-active-review-")); + try { + const projectRoot = await realpath(root); + for (const drift of [false, true]) { + const f = fixture(); + const scoped = { + ...scope, + projectRoot, + projectDir: join(projectRoot, ".hack"), + branch: "feature-a", + }; + const calls: string[] = []; + let compatible = false; + const input = { + originalSha256: "1".repeat(64), + environmentFiles: [], + serviceNames: ["app"], + normalizedComposeJson: JSON.stringify({ + services: { + app: { + image: `sha256:${"a".repeat(64)}`, + labels: { + caddy: "app.hack, app.hack.gy", + "caddy.tls": "internal", + "caddy.reverse_proxy": "{{upstreams 3000}}", + }, + }, + }, + }), + managedEnvironment: {}, + lifecycleHostEnvironment: {}, + effectiveEnvName: "qa", + }; + const result = restartNativeProject({ + ...f.options, + scope: scoped, + preflight: async () => + await preflightNativeRestart({ + runtime: { binary: "/unused", home: "/candidate" }, + scope: scoped, + composeFile: join(projectRoot, "compose.yml"), + run, + dependencies: { + prepare: async () => input, + adapt: async ({ input: value }) => value, + dependencies: async () => [], + invoke: async ({ args }) => { + calls.push(args[1] ?? "unknown"); + if (args[1] === "status") { + return { network: "internet" }; + } + if (args[1] === "probe") { + return { admitted: true }; + } + if (args[0] === "project") { + if (args.includes("--normalized-file")) { + const value = JSON.parse( + await readFile( + args[args.indexOf("--normalized-file") + 1] ?? "", + "utf8" + ) + ); + expect(value.services.app.labels.caddy).toBe( + "app.hack, app.hack.gy" + ); + expect(args).not.toContain("--branch"); + } + return { + plan_id: "e".repeat(64), + plan: { + source: projectRoot, + namespace: args.includes("--branch") + ? "f".repeat(64) + : run.namespace, + compose_sha256: input.originalSha256, + services: { app: { active: true } }, + }, + }; + } + if (args[1] === "run-selection") { + expect(args[args.indexOf("--service") + 1]).toBe("app"); + return { + ok: true, + run: run.run, + owner: run.owner, + namespace: run.namespace, + plan: run.planId, + service: "app", + container: "2".repeat(64), + boot: compatible && drift ? "changed-boot" : "owned-boot", + generation: "3".repeat(64), + }; + } + if (args[1] === "source-compatibility") { + compatible = true; + return { + run: run.run, + owner: run.owner, + namespace: run.namespace, + plan: run.planId, + reviewed_plan: "e".repeat(64), + source_revision: null, + }; + } + throw new Error("Unexpected effect"); + }, + }, + }), + }); + if (drift) { + await expect(result).rejects.toThrow("active review identity changed"); + expect(f.events).toEqual([]); + } else { + expect(await result).toBe(0); + expect(f.events).toContain("down"); + } + expect(calls.at(-1)).toBe("run-selection"); + expect(calls.filter((action) => action === "run-selection")).toHaveLength( + 3 + ); + expect(calls).not.toContain("restore-selection"); + expect(calls).not.toContain("run-service"); + } + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/tests/native-project-review.test.ts b/tests/native-project-review.test.ts index 1a3de8aa8..4e5d52d0e 100644 --- a/tests/native-project-review.test.ts +++ b/tests/native-project-review.test.ts @@ -7,6 +7,7 @@ import { selectNativeProjectRestore } from "../src/backends/native-project-resto import { prepareNativeReviewBranch, selectNativeProjectReviewIdentity, + verifyNativeActiveReview, withNativeProjectReview, } from "../src/backends/native-project-review.ts"; import type { invokeNativeRuntime } from "../src/backends/native-runtime-client.ts"; @@ -154,11 +155,16 @@ async function legacyFixture() { namespace: retained.namespace, plan: retained.planId, generation: "f".repeat(64), + ok: true, + service: "web", + container: "1".repeat(64), + boot: "owned-boot", }; const unbranched: Record = { namespace: retained.namespace, source: await realpath(options.projectRoot), compose_sha256: options.input.originalSha256, + services: { web: { active: true } }, }; const invoke: typeof invokeNativeRuntime = async ({ args }) => { calls.push([...args]); @@ -375,29 +381,37 @@ test("early identity proof is rechecked after route preparation before normalize } }); -test("active branch-native review needs no stopped selection; active legacy mismatch refuses before cleanup", async () => { +test("active branch-native needs no stopped selection and legacy selects authenticated service authority", async () => { const { options, calls, invoke } = await legacyFixture(); - const identity = await selectNativeProjectReviewIdentity({ + const scoped = await selectNativeProjectReviewIdentity({ ...options, branch: "feature-a", retained: { ...retained, namespace: "c".repeat(64) }, retainedMode: "active", invoke, }); - expect(identity.branch).toBe("feature-a"); + expect(scoped.branch).toBe("feature-a"); expect(calls).toHaveLength(1); calls.splice(0); - await expect( - selectNativeProjectReviewIdentity({ - ...options, - branch: "feature-a", - retained, - retainedMode: "active", - invoke, - }) - ).rejects.toThrow("Active legacy native branch review"); - expect(calls).toHaveLength(1); - expect(calls[0]?.[0]).toBe("project"); + const identity = await selectNativeProjectReviewIdentity({ + ...options, + branch: "feature-a", + retained, + retainedMode: "active", + invoke, + }); + expect(identity.branch).toBeNull(); + expect(identity.retainedGeneration).toBeUndefined(); + expect(identity.activeProof?.service).toBe("web"); + await verifyNativeActiveReview({ + ...options, + retained, + proof: identity.activeProof, + invoke, + }); + expect( + calls.filter((args) => args[0] === "graph").map((args) => args[1]) + ).toEqual(["run-selection", "run-selection"]); }); test("deferred retained review rejects malformed branch before hooks or runtime selection", async () => { @@ -418,3 +432,83 @@ test("deferred retained review rejects malformed branch before hooks or runtime ).rejects.toThrow("canonical branch"); expect(calls).toHaveLength(0); }); + +test("active proof refuses every tuple drift after compatibility and unavailable service authority", async () => { + for (const key of [ + "run", + "owner", + "namespace", + "plan", + "service", + "container", + "boot", + "generation", + ]) { + const { options, selected, invoke } = await legacyFixture(); + const identity = await selectNativeProjectReviewIdentity({ + ...options, + branch: "feature-a", + retained, + retainedMode: "active", + invoke, + }); + selected[key] = + key === "boot" + ? "another-boot" + : "2".repeat(key === "run" || key === "owner" ? 32 : 64); + await expect( + verifyNativeActiveReview({ + ...options, + retained, + proof: identity.activeProof, + invoke, + }) + ).rejects.toThrow("changed"); + } + for (const services of [{}, { web: { active: false } }]) { + const { options, calls, unbranched, invoke } = await legacyFixture(); + unbranched.services = services; + await expect( + selectNativeProjectReviewIdentity({ + ...options, + branch: "feature-a", + retained, + retainedMode: "active", + invoke, + }) + ).rejects.toThrow("no supported service authority"); + expect(calls).toHaveLength(1); + } +}); + +test("active legacy proof never grants an unrelated source and propagates stale dependency refusal", async () => { + for (const failure of ["source", "dependencies"]) { + const { options, invoke, unbranched, calls } = await legacyFixture(); + const checked: typeof invokeNativeRuntime = async (request) => { + if (failure === "dependencies" && request.args[1] === "run-selection") { + throw new Error("stale dependency"); + } + if (failure === "source") { + unbranched.source = "/foreign/project"; + } + return await invoke(request); + }; + await expect( + selectNativeProjectReviewIdentity({ + ...options, + branch: "feature-a", + retained, + retainedMode: "active", + invoke: checked, + }) + ).rejects.toThrow( + failure === "source" ? "review changed" : "stale dependency" + ); + expect( + calls.some( + (args) => + args[1] === "run-service" || args.includes("--normalized-file") + ) + ).toBe(false); + } +}); From 553515287ef84fc801c216fd4010f2b11cdff35b Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 09:26:32 -0400 Subject: [PATCH 18/25] fix(cli): retain content IDs when restoring unchanged image declarations --- docs/guides/native-candidate.md | 6 + .../native-project-retained-images.ts | 84 +++++++++ src/backends/native-project-start.ts | 77 +++++--- tests/native-project-retained-images.test.ts | 165 ++++++++++++++++++ tests/native-project-start.test.ts | 63 +++++++ 5 files changed, 373 insertions(+), 22 deletions(-) create mode 100644 src/backends/native-project-retained-images.ts create mode 100644 tests/native-project-retained-images.test.ts diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 4211e6179..81887c104 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -360,6 +360,12 @@ and route selections. Normal foreground `hack restart` performs this selection and retained-data restore. It verifies the old containers and networks are absent and the retained volumes still have their recorded identities; it does not silently create replacement data. +When the original Compose file is unchanged, retained startup also selects the +authenticated container image IDs from that stopped graph. Mutable tags are not +resolved again for those services. Missing images still refuse native admission; +this selection never replaces generation, source or resource checks. Changing the +original Compose file uses normal image resolution and the existing compatibility +rules, rather than silently adopting an old image for a new declaration. For a shared-source graph admitted with a retained compatibility contract, ordinary source-content edits may change the reviewed plan ID: restart checks the stable execution, exclusion, mount and dependency-cache inputs before cleanup and again diff --git a/src/backends/native-project-retained-images.ts b/src/backends/native-project-retained-images.ts new file mode 100644 index 000000000..f84b2336b --- /dev/null +++ b/src/backends/native-project-retained-images.ts @@ -0,0 +1,84 @@ +import { isRecord } from "../lib/guards.ts"; +import { inspectNativeProjectGraph } from "./native-project-inspect.ts"; +import { confirmedNativeRetainedGraph } from "./native-project-retained.ts"; +import type { NativeProjectRun } from "./native-project-run.ts"; +import type { + invokeNativeRuntime, + NativeRuntimeSelection, +} from "./native-runtime-client.ts"; + +const SHA = /^[a-f0-9]{64}$/; +const IMAGE = /^sha256:[a-f0-9]{64}$/; + +/** + * Restore unchanged declarations with their authenticated content IDs. Resolving + * a mutable tag again could change compute around retained data. This observation + * grants no restore authority: generation, input, source and resource checks still + * run in native code at admission. Changed original input keeps normal resolution. + */ +export async function selectNativeRetainedImages(opts: { + readonly runtime: NativeRuntimeSelection; + readonly projectRoot: string; + readonly originalSha256: string; + readonly restore?: NativeProjectRun; + readonly invoke?: typeof invokeNativeRuntime; +}): Promise> { + if (!opts.restore) { + return new Map(); + } + const observed = await inspectNativeProjectGraph({ + ...opts, + run: opts.restore.run, + }); + if ( + !( + confirmedNativeRetainedGraph(observed, opts.restore) && + isRecord(observed) && + isRecord(observed.receipt) + ) + ) { + throw new Error( + "Native retained image selection changed; no images were resolved." + ); + } + const { receipt } = observed; + if (receipt.normalized_input === undefined) { + return new Map(); + } + const normalized = receipt.normalized_input; + if ( + !isRecord(normalized) || + normalized.namespace !== opts.restore.namespace || + typeof normalized.original_compose_sha256 !== "string" || + !SHA.test(normalized.original_compose_sha256) || + typeof normalized.normalized_compose_sha256 !== "string" || + !SHA.test(normalized.normalized_compose_sha256) || + !SHA.test(opts.originalSha256) || + !isRecord(receipt.resources) + ) { + throw new Error( + "Native retained image provenance is invalid; no images were resolved." + ); + } + if (normalized.original_compose_sha256 !== opts.originalSha256) { + return new Map(); + } + const images = new Map(); + for (const [key, resource] of Object.entries(receipt.resources)) { + if (!isRecord(resource) || resource.kind !== "container") { + continue; + } + if ( + typeof resource.key !== "string" || + key !== `container:${resource.key}` || + typeof resource.image !== "string" || + !IMAGE.test(resource.image) + ) { + throw new Error( + "Native retained image identity is invalid; no images were resolved." + ); + } + images.set(resource.key, resource.image); + } + return images; +} diff --git a/src/backends/native-project-start.ts b/src/backends/native-project-start.ts index 3f0ec4838..124669d96 100644 --- a/src/backends/native-project-start.ts +++ b/src/backends/native-project-start.ts @@ -34,6 +34,7 @@ import { validateNativeAllowedHosts } from "./native-project-network.ts"; import { serveNativeProjectGraph } from "./native-project-process.ts"; import { selectNativeProjectRestore } from "./native-project-restore.ts"; import { confirmedNativeRetainedGraph } from "./native-project-retained.ts"; +import { selectNativeRetainedImages } from "./native-project-retained-images.ts"; import { preflightNativeRetainedStartup, verifyNativeResumedRetainedGraph, @@ -215,6 +216,7 @@ type Dependencies = { adaptAws: typeof adaptNativeAwsEnvironment; review: typeof withNativeProjectReview; selectReview: typeof selectNativeProjectReviewIdentity; + retainedImages: typeof selectNativeRetainedImages; serve: typeof serveNativeProjectGraph; https: typeof acquireNativeHttpsLease; recoverHttps: typeof recoverNativeHttpsLease; @@ -231,6 +233,7 @@ const DEFAULTS: Dependencies = { adaptAws: adaptNativeAwsEnvironment, review: withNativeProjectReview, selectReview: selectNativeProjectReviewIdentity, + retainedImages: selectNativeRetainedImages, serve: serveNativeProjectGraph, https: acquireNativeHttpsLease, recoverHttps: recoverNativeHttpsLease, @@ -278,6 +281,43 @@ export function prepareNativeProjectServices( } return result; } +/** Keep image acquisition separate from graph admission and preserve cancellation between requests. */ +async function pinNativeProjectImages(opts: { + readonly runtime: NativeRuntimeSelection; + readonly projectRoot: string; + readonly specs: Record>; + readonly retainedImages: ReadonlyMap; + readonly signal: AbortSignal; + readonly invoke: typeof invokeNativeRuntime; +}): Promise { + for (const [name, spec] of Object.entries(opts.specs)) { + if (opts.signal.aborted) { + throw refused(); + } + const retainedImage = opts.retainedImages.get(name); + if (retainedImage) { + spec.image = retainedImage; + continue; + } + const image = String(spec.image); + if (IMAGE.test(image)) { + continue; + } + const ensured = await opts.invoke({ + runtime: opts.runtime, + cwd: opts.projectRoot, + args: ["runtime", "ensure-image", "--reference", image, "--json"], + }); + if ( + !isRecord(ensured) || + typeof ensured.image_id !== "string" || + !IMAGE.test(ensured.image_id) + ) { + throw refused(); + } + spec.image = ensured.image_id; + } +} function readiness( specs: Record>, initializers: ReadonlySet, @@ -886,28 +926,21 @@ export async function startNativeProject(opts: { dependencies: hostDependencies, services: specs, }); - for (const spec of Object.values(specs)) { - if (controller.signal.aborted) { - throw refused(); - } - const image = String(spec.image); - if (IMAGE.test(image)) { - continue; - } - const ensured = await deps.invoke({ - runtime: opts.runtime, - cwd: opts.scope.projectRoot, - args: ["runtime", "ensure-image", "--reference", image, "--json"], - }); - if ( - !isRecord(ensured) || - typeof ensured.image_id !== "string" || - !IMAGE.test(ensured.image_id) - ) { - throw refused(); - } - spec.image = ensured.image_id; - } + const retainedImages = await deps.retainedImages({ + runtime: opts.runtime, + projectRoot: opts.scope.projectRoot, + originalSha256: input.originalSha256, + restore, + invoke: deps.invoke, + }); + await pinNativeProjectImages({ + runtime: opts.runtime, + projectRoot: opts.scope.projectRoot, + specs, + retainedImages, + signal: controller.signal, + invoke: deps.invoke, + }); const compose = JSON.parse(input.normalizedComposeJson); compose.services = specs; const pinned = { ...input, normalizedComposeJson: JSON.stringify(compose) }; diff --git a/tests/native-project-retained-images.test.ts b/tests/native-project-retained-images.test.ts new file mode 100644 index 000000000..0aef126b5 --- /dev/null +++ b/tests/native-project-retained-images.test.ts @@ -0,0 +1,165 @@ +import { expect, test } from "bun:test"; +import { selectNativeRetainedImages } from "../src/backends/native-project-retained-images.ts"; + +const saved = { + run: "a".repeat(32), + owner: "b".repeat(32), + namespace: "c".repeat(64), + planId: "d".repeat(64), +}; +const image = `sha256:${"e".repeat(64)}`; +const originalSha256 = "f".repeat(64); +const options = { + runtime: { binary: "/not-invoked", home: "/fixture" }, + projectRoot: "/fixture/project", + originalSha256, + restore: saved, +}; +function observation() { + return { + journal_incomplete: false, + receipt: { + run: saved.run, + owner: saved.owner, + namespace: saved.namespace, + plan_id: saved.planId, + phase: "stopped-data-retained", + normalized_input: { + namespace: saved.namespace, + original_compose_sha256: originalSha256, + normalized_compose_sha256: "1".repeat(64), + }, + resources: { + "container:web": { kind: "container", key: "web", image }, + "volume:data": { kind: "volume", key: "data" }, + }, + }, + observations: { + "container:web": { state: "absent" }, + "volume:data": { state: "present" }, + }, + }; +} +test("unchanged retained input selects verified content IDs without resolving tags", async () => { + const calls: string[][] = []; + const selected = await selectNativeRetainedImages({ + ...options, + invoke: async ({ args }) => { + calls.push([...args]); + return observation(); + }, + }); + expect([...selected]).toEqual([["web", image]]); + expect(calls).toEqual([ + ["graph", "inspect", "--run-id", saved.run, "--json"], + ]); +}); +test("fresh input never observes retained state", async () => { + expect( + ( + await selectNativeRetainedImages({ + ...options, + restore: undefined, + invoke: async () => { + throw new Error("unexpected access"); + }, + }) + ).size + ).toBe(0); +}); +test("edited original input and legacy absence keep ordinary resolution", async () => { + expect( + ( + await selectNativeRetainedImages({ + ...options, + originalSha256: "2".repeat(64), + invoke: async () => observation(), + }) + ).size + ).toBe(0); + const { normalized_input: _normalized, ...legacy } = observation().receipt; + expect( + ( + await selectNativeRetainedImages({ + ...options, + invoke: async () => ({ ...observation(), receipt: legacy }), + }) + ).size + ).toBe(0); +}); +test("stale ownership and uncertain compute or data refuse image reuse", async () => { + const baseline = observation(); + for (const changed of [ + { ...baseline, journal_incomplete: true }, + ...["run", "owner", "namespace", "plan_id", "phase"].map((key) => ({ + ...baseline, + receipt: { ...baseline.receipt, [key]: "changed" }, + })), + { + ...baseline, + observations: { + ...baseline.observations, + "container:web": { state: "running" }, + }, + }, + { + ...baseline, + observations: { + ...baseline.observations, + "volume:data": { state: "absent" }, + }, + }, + ]) { + await expect( + selectNativeRetainedImages({ ...options, invoke: async () => changed }) + ).rejects.toThrow("image selection changed"); + } +}); +test("malformed provenance or image and mismatched resource keys refuse reuse", async () => { + const baseline = observation(); + for (const field of [ + "namespace", + "original_compose_sha256", + "normalized_compose_sha256", + ]) { + await expect( + selectNativeRetainedImages({ + ...options, + invoke: async () => ({ + ...baseline, + receipt: { + ...baseline.receipt, + normalized_input: { + ...baseline.receipt.normalized_input, + [field]: "invalid", + }, + }, + }), + }) + ).rejects.toThrow("provenance is invalid"); + } + for (const resource of [ + { ...baseline.receipt.resources["container:web"], image: "redis:latest" }, + { ...baseline.receipt.resources["container:web"], key: "worker" }, + ]) { + await expect( + selectNativeRetainedImages({ + ...options, + invoke: async () => ({ + ...baseline, + receipt: { + ...baseline.receipt, + resources: { + ...baseline.receipt.resources, + "container:web": resource, + }, + }, + }), + }) + ).rejects.toThrow( + resource.key === "web" + ? "image identity is invalid" + : "image selection changed" + ); + } +}); diff --git a/tests/native-project-start.test.ts b/tests/native-project-start.test.ts index 542d7cdaf..b7617ea6c 100644 --- a/tests/native-project-start.test.ts +++ b/tests/native-project-start.test.ts @@ -13,6 +13,7 @@ import type { acquireNativeHttpsLease } from "../src/backends/native-https-owner import { beginNativeProjectFinalization } from "../src/backends/native-project-finalization.ts"; import type { NativeProjectInput } from "../src/backends/native-project-input.ts"; import { preflightNativeRestart } from "../src/backends/native-project-restart-preflight.ts"; +import { selectNativeRetainedImages } from "../src/backends/native-project-retained-images.ts"; import type { NativeProjectRun } from "../src/backends/native-project-run.ts"; import { parseNativeHttpsSelection, @@ -50,6 +51,7 @@ async function fixture(withEnvironment = true) { Parameters[0]["dependencies"] > = { prepareStorage: async () => {}, + retainedImages: async () => new Map(), load: async () => null, loadRestart: async () => null, save: async () => { @@ -2050,3 +2052,64 @@ test("retained startup selects legacy labels only after runtime admission and be } } }); + +test("retained startup reviews the saved content ID instead of a newly resolved mutable tag", async () => { + const { opts, events } = await fixture(false); + const saved = { + run: "1".repeat(32), + owner: "c".repeat(32), + namespace: "b".repeat(64), + planId: "a".repeat(64), + }; + const image = `sha256:${"e".repeat(64)}`; + const stopped = stoppedGraph(saved); + const observed = { + ...stopped, + receipt: { + ...stopped.receipt, + normalized_input: { + namespace: saved.namespace, + original_compose_sha256: "d".repeat(64), + normalized_compose_sha256: "f".repeat(64), + }, + resources: { + "container:web": { + ...stopped.receipt.resources["container:web"], + image, + }, + }, + }, + }; + opts.dependencies.retainedImages = selectNativeRetainedImages; + const prepare = opts.dependencies.prepare!; + opts.dependencies.prepare = async (request) => ({ + ...(await prepare(request)), + normalizedComposeJson: JSON.stringify({ + services: { web: { image: "redis:latest" } }, + }), + }); + const invoke = opts.dependencies.invoke!; + let beforeReview = true; + opts.dependencies.invoke = async (request) => { + if (request.args[1] === "ensure-image") { + throw new Error("mutable tag was unexpectedly resolved"); + } + if (request.args[1] === "inspect" && beforeReview) { + expect(events).toContain("runtime up"); + return observed; + } + if (request.args[1] === "restore-selection") { + return { ...saved, plan: saved.planId, generation: "2".repeat(64) }; + } + return await invoke(request); + }; + const review = opts.dependencies.review!; + opts.dependencies.review = async (request) => { + beforeReview = false; + expect( + JSON.parse(request.input.normalizedComposeJson).services.web.image + ).toBe(image); + return await review(request); + }; + expect(await startNativeProject({ ...opts, restore: saved })).toBe(0); +}); From f90bdd84131bb324bebca088f20d7873247ce67f Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 10:34:37 -0400 Subject: [PATCH 19/25] fix(runtime): recover selected dependency sockets in a quiescent live pool --- packages/runtime-core/README.md | 15 + packages/runtime-core/src/main.rs | 30 ++ .../src/provider/dependency_socket.rs | 1 + .../dependency_socket/quiescent_recovery.rs | 267 +++++++++++ .../quiescent_recovery_tests.rs | 213 ++++++++ .../provider/dependency_socket/recovery.rs | 12 +- .../src/provider/graph/bridges/cleanup.rs | 12 + .../provider/graph/foreground/transport.rs | 5 + .../graph/foreground/transport_tests.rs | 37 ++ .../runtime-core/src/provider/graph/mod.rs | 4 + .../src/provider/graph/publication_gate.rs | 50 ++ .../graph/quiescent_dependency_recovery.rs | 453 ++++++++++++++++++ packages/runtime-core/src/provider/mod.rs | 1 + ...escent_dependency_recovery_cli_contract.rs | 60 +++ 14 files changed, 1154 insertions(+), 6 deletions(-) create mode 100644 packages/runtime-core/src/provider/dependency_socket/quiescent_recovery.rs create mode 100644 packages/runtime-core/src/provider/dependency_socket/quiescent_recovery_tests.rs create mode 100644 packages/runtime-core/src/provider/graph/publication_gate.rs create mode 100644 packages/runtime-core/src/provider/graph/quiescent_dependency_recovery.rs create mode 100644 packages/runtime-core/tests/quiescent_dependency_recovery_cli_contract.rs diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 15886c72a..9444e009b 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -259,6 +259,21 @@ state or changed provider identity refuses recovery. VM disks and graph data rem This is cooperative same-user recovery for a stopped pool, not adoption of an unreceipted active relay or proof of application readiness. +For an already-running VM whose retained graphs are all stopped, the separate +`hack-local runtime quiescent-dependency-socket-recovery --json` command selects +legacy unreceipted socket inodes without rebooting. Explicitly apply its selection +with `hack-local runtime recover-quiescent-dependency-sockets --expect-sha256 --json`. +This mode holds a pool publication gate before foreground retirement locks and the +provider lease. It pins the exact provider receipt, host/guest boot, every retained +graph receipt and volume identity, and requires absent compute, no foreground +publisher, no dependency assignment and no untracked guest containers. It rechecks +that proof before each inode-selected unlink and journals the selection separately +from stopped-pool recovery so partial retries retain their original scope. +An incomplete journal, live listener, replacement path or changed proof refuses +recovery. Owner bytes, source-rebind witnesses, retained data and the running VM +remain unchanged. This is explicit cooperative legacy cleanup, not proof of the +original socket creator or a security boundary against another same-user process. + TERM and INT are checked during initial graph startup, including readiness waits and before new service effects. Cancellation enters owned cleanup while preserving persistent data. Checks occur between bounded operations; an in-flight operation diff --git a/packages/runtime-core/src/main.rs b/packages/runtime-core/src/main.rs index b4c53ad58..a7434f749 100644 --- a/packages/runtime-core/src/main.rs +++ b/packages/runtime-core/src/main.rs @@ -59,6 +59,8 @@ Usage: hack-local runtime publication-recovery [--json] hack-local runtime recover-publications --expect-sha256 [--json] hack-local runtime dependency-socket-recovery [--json] + hack-local runtime quiescent-dependency-socket-recovery [--json] + hack-local runtime recover-quiescent-dependency-sockets --expect-sha256 [--json] hack-local runtime recover-dependency-sockets --expect-sha256 [--json] hack-local runtime bridge-recovery [--json] hack-local runtime export-bridge-recovery --slot <1..8> --expect-sha256 [--json] @@ -680,6 +682,34 @@ fn run() -> Result<(), CandidateError> { )?, )?; } + ["runtime", "quiescent-dependency-socket-recovery"] + | ["runtime", "quiescent-dependency-socket-recovery", "--json"] => { + print_json( + &hack_runtime_core::provider::quiescent_dependency_socket_recovery::inspect( + &discover_candidate(&requested)?, + )?, + )?; + } + [ + "runtime", + "recover-quiescent-dependency-sockets", + "--expect-sha256", + hash, + ] + | [ + "runtime", + "recover-quiescent-dependency-sockets", + "--expect-sha256", + hash, + "--json", + ] => { + print_json( + &hack_runtime_core::provider::quiescent_dependency_socket_recovery::recover( + &discover_candidate(&requested)?, + hash, + )?, + )?; + } [ "runtime", "recover-dependency-sockets", diff --git a/packages/runtime-core/src/provider/dependency_socket.rs b/packages/runtime-core/src/provider/dependency_socket.rs index 0b2bfd157..071cac20f 100644 --- a/packages/runtime-core/src/provider/dependency_socket.rs +++ b/packages/runtime-core/src/provider/dependency_socket.rs @@ -9,6 +9,7 @@ use std::{ path::{Path, PathBuf}, }; +pub mod quiescent_recovery; pub mod recovery; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/packages/runtime-core/src/provider/dependency_socket/quiescent_recovery.rs b/packages/runtime-core/src/provider/dependency_socket/quiescent_recovery.rs new file mode 100644 index 000000000..f2b154a72 --- /dev/null +++ b/packages/runtime-core/src/provider/dependency_socket/quiescent_recovery.rs @@ -0,0 +1,267 @@ +//! Explicit legacy socket retirement while an independently verified VM stays live. +//! This uses a separate receipt from stopped-pool recovery. It never proves who +//! created an unreceipted socket; the reviewed selection authorizes only those inodes. +#[cfg(any(test, target_os = "macos"))] +use super::{ + DependencySocketIntent, + recovery::{Socket, digest, hex, observed as observe_socket, path}, +}; +#[cfg(any(test, target_os = "macos"))] +use crate::provider::state; +use crate::{Candidate, CandidateError}; +#[cfg(any(test, target_os = "macos"))] +use serde::{Deserialize, Serialize}; +use serde_json::Value; +#[cfg(any(test, target_os = "macos"))] +use serde_json::json; +#[cfg(any(test, target_os = "macos"))] +use std::{ + fs::{self, File}, + path::{Path, PathBuf}, +}; + +#[cfg(any(test, target_os = "macos"))] +const MAX_RECEIPTS: usize = 16; +#[cfg(any(test, target_os = "macos"))] +const RECEIPT_LIMIT: u64 = 16 * 1024; + +fn refused() -> CandidateError { + CandidateError::new( + "quiescent_dependency_socket_recovery", + "Live pool quiescence or selected socket identity is uncertain; paths and data were retained.", + ) +} + +#[cfg(any(test, target_os = "macos"))] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Scope { + version: u8, + quiescence_sha256: String, + slots: u8, +} + +#[cfg(any(test, target_os = "macos"))] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Selection { + scope: Scope, + sockets: Vec, +} + +#[cfg(target_os = "macos")] +fn journal(candidate: &Candidate) -> PathBuf { + candidate + .state_root + .join("run/smolvm/quiescent-dependency-socket-recovery") +} + +#[cfg(any(test, target_os = "macos"))] +fn observed(directory: &Path, slot: u8) -> Result, CandidateError> { + observe_socket(directory, slot).map_err(|_| refused()) +} + +#[cfg(any(test, target_os = "macos"))] +fn current(scope: Scope, directory: &Path) -> Result { + DependencySocketIntent::new(scope.slots)?; + if scope.version != 1 || !hex(&scope.quiescence_sha256) { + return Err(refused()); + } + let mut sockets = Vec::new(); + for slot in 0..scope.slots { + if let Some(socket) = observed(directory, slot)? { + sockets.push(socket); + } + } + Ok(Selection { scope, sockets }) +} + +#[cfg(any(test, target_os = "macos"))] +fn matching( + selection: &Selection, + scope: &Scope, + directory: &Path, +) -> Result { + if selection.scope != *scope + || selection.sockets.is_empty() + || selection.sockets.len() > usize::from(scope.slots) + || selection.sockets.iter().any(|s| s.slot >= scope.slots) + || selection + .sockets + .windows(2) + .any(|pair| pair[0].slot >= pair[1].slot) + { + return Err(refused()); + } + let mut remaining = 0; + for slot in 0..scope.slots { + let saved = selection.sockets.iter().find(|item| item.slot == slot); + match (saved, observed(directory, slot)?) { + (Some(saved), Some(actual)) if *saved == actual => remaining += 1, + (Some(_), None) | (None, None) => {} + _ => return Err(refused()), + } + } + Ok(remaining) +} + +#[cfg(any(test, target_os = "macos"))] +fn receipts(root: &Path) -> Result, CandidateError> { + let entries = match fs::read_dir(root) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(_) => return Err(refused()), + }; + state::check_private_directory(root)?; + let mut paths = Vec::new(); + for entry in entries { + let entry = entry.map_err(|_| refused())?; + let name = entry.file_name().into_string().map_err(|_| refused())?; + // An incomplete journal or unknown file is retained and never interpreted + // as permission to delete. Only committed, bounded selections can resume. + if !name.strip_suffix(".json").is_some_and(hex) || paths.len() >= MAX_RECEIPTS { + return Err(refused()); + } + paths.push(entry.path()); + } + paths.sort(); + Ok(paths) +} + +#[cfg(any(test, target_os = "macos"))] +fn read(root: &Path, hash: &str) -> Result { + let selected: Selection = + state::read_bounded(&root.join(format!("{hash}.json")), RECEIPT_LIMIT)?; + if digest(&selected)? != hash { + return Err(refused()); + } + Ok(selected) +} + +#[cfg(any(test, target_os = "macos"))] +fn inspect_scope(root: &Path, scope: &Scope, directory: &Path) -> Result { + for receipt in receipts(root)? { + let hash = receipt + .file_stem() + .and_then(|s| s.to_str()) + .ok_or_else(refused)?; + let selected = read(root, hash)?; + if selected.scope == *scope { + let remaining = matching(&selected, scope, directory)?; + if remaining > 0 { + return Ok( + json!({"recoverable":true,"sha256":hash,"selected":selected.sockets.len(),"remaining":remaining,"resumable":true,"mode":"live-quiescent-legacy"}), + ); + } + } + } + let selected = current(scope.clone(), directory)?; + if selected.sockets.is_empty() { + return Ok(json!({"recoverable":false,"selected":0,"mode":"live-quiescent-legacy"})); + } + Ok( + json!({"recoverable":true,"sha256":digest(&selected)?,"selected":selected.sockets.len(),"remaining":selected.sockets.len(),"resumable":false,"mode":"live-quiescent-legacy"}), + ) +} + +#[cfg(any(test, target_os = "macos"))] +fn recover_scope( + root: &Path, + scope: &Scope, + directory: &Path, + expected: &str, + verify: impl Fn() -> Result<(), CandidateError>, +) -> Result { + if !hex(expected) { + return Err(refused()); + } + verify()?; + let paths = receipts(root)?; + let filename = root.join(format!("{expected}.json")); + let saved = if paths.contains(&filename) { + read(root, expected)? + } else { + let selected = current(scope.clone(), directory)?; + if selected.sockets.is_empty() + || digest(&selected)? != expected + || paths.len() >= MAX_RECEIPTS + { + return Err(refused()); + } + verify()?; + matching(&selected, scope, directory)?; + state::private_directory(root)?; + state::write(&filename, &selected)?; + selected + }; + let remaining = matching(&saved, scope, directory)?; + for socket in &saved.sockets { + verify()?; + matching(&saved, scope, directory)?; + if observed(directory, socket.slot)?.as_ref() == Some(socket) { + fs::remove_file(path(directory, socket.slot)).map_err(state::io)?; + File::open(directory) + .and_then(|f| f.sync_all()) + .map_err(state::io)?; + } + } + verify()?; + if matching(&saved, scope, directory)? != 0 { + return Err(refused()); + } + Ok( + json!({"recovered":true,"sha256":expected,"selected":saved.sockets.len(),"removed":remaining,"already_absent":saved.sockets.len()-remaining,"data_retained":true,"mode":"live-quiescent-legacy"}), + ) +} + +/// Select unlistened declared socket inodes only after holding the pool publication +/// gate, all retained graph retirement locks and the provider lease. Owner bytes, +/// guest boot, data volumes, empty reservations and absent compute are pinned. +#[cfg(target_os = "macos")] +pub fn inspect(candidate: &Candidate) -> Result { + let guard = crate::provider::graph::quiescent_dependency_recovery::Guard::acquire(candidate)?; + let scope = Scope { + version: 1, + quiescence_sha256: guard.sha256().into(), + slots: guard.owner().dependency_sockets.ok_or_else(refused)?.slots, + }; + let result = inspect_scope(&journal(candidate), &scope, &guard.owner().short_home)?; + guard.verify(candidate)?; + Ok(result) +} + +/// Explicit inode-selected recovery. A committed, separate journal precedes every +/// unlink; each effect rechecks the held quiescence proof. A partial retry accepts +/// exact remaining inodes or absence, and never restarts the VM or edits its Owner. +#[cfg(target_os = "macos")] +pub fn recover(candidate: &Candidate, expected: &str) -> Result { + if !hex(expected) { + return Err(refused()); + } + let guard = crate::provider::graph::quiescent_dependency_recovery::Guard::acquire(candidate)?; + let scope = Scope { + version: 1, + quiescence_sha256: guard.sha256().into(), + slots: guard.owner().dependency_sockets.ok_or_else(refused)?.slots, + }; + recover_scope( + &journal(candidate), + &scope, + &guard.owner().short_home, + expected, + || guard.verify(candidate), + ) +} + +#[cfg(not(target_os = "macos"))] +pub fn inspect(_: &Candidate) -> Result { + Err(refused()) +} +#[cfg(not(target_os = "macos"))] +pub fn recover(_: &Candidate, _: &str) -> Result { + Err(refused()) +} + +#[cfg(test)] +#[path = "quiescent_recovery_tests.rs"] +mod tests; diff --git a/packages/runtime-core/src/provider/dependency_socket/quiescent_recovery_tests.rs b/packages/runtime-core/src/provider/dependency_socket/quiescent_recovery_tests.rs new file mode 100644 index 000000000..8edfbe567 --- /dev/null +++ b/packages/runtime-core/src/provider/dependency_socket/quiescent_recovery_tests.rs @@ -0,0 +1,213 @@ +use super::*; +use std::{ + cell::Cell, + os::unix::{fs::PermissionsExt, net::UnixListener}, + sync::atomic::{AtomicU64, Ordering}, +}; + +static NEXT: AtomicU64 = AtomicU64::new(0); +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + let base = if cfg!(target_os = "macos") { + PathBuf::from("/private/tmp") + } else { + std::env::temp_dir() + }; + let root = base.join(format!( + "hk-qd-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&root).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); + Self(root) + } + fn journal(&self) -> PathBuf { + self.0.join("journal") + } + fn scope(&self) -> Scope { + Scope { + version: 1, + quiescence_sha256: "a".repeat(64), + slots: 2, + } + } + fn bind(&self, slot: u8) -> UnixListener { + let target = path(&self.0, slot); + let listener = UnixListener::bind(&target).unwrap(); + fs::set_permissions(target, fs::Permissions::from_mode(0o600)).unwrap(); + listener + } + fn stale(&self, slot: u8) { + drop(self.bind(slot)); + } + fn selection(&self) -> Selection { + current(self.scope(), &self.0).unwrap() + } + fn hash(&self) -> String { + digest(&self.selection()).unwrap() + } + fn recover(&self, hash: &str) -> Result { + recover_scope(&self.journal(), &self.scope(), &self.0, hash, || Ok(())) + } +} +impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_dir_all(&self.0).unwrap(); + } +} + +#[test] +fn selected_recovery_journals_before_unlink_and_is_idempotent() { + let fixture = Fixture::new(); + fixture.stale(0); + fixture.stale(1); + fs::write(fixture.0.join("retained-data"), b"unchanged").unwrap(); + let hash = fixture.hash(); + assert_eq!(fixture.recover(&hash).unwrap()["removed"], 2); + let receipt = fixture.journal().join(format!("{hash}.json")); + let bytes = fs::read(&receipt).unwrap(); + let retry = fixture.recover(&hash).unwrap(); + assert_eq!(retry["removed"], 0); + assert_eq!(retry["already_absent"], 2); + assert_eq!(fs::read(receipt).unwrap(), bytes); + assert_eq!( + fs::read(fixture.0.join("retained-data")).unwrap(), + b"unchanged" + ); + assert_eq!( + inspect_scope(&fixture.journal(), &fixture.scope(), &fixture.0).unwrap()["recoverable"], + false + ); +} + +#[test] +fn proof_drift_between_unlinks_stops_and_resumes_only_exact_remaining_inodes() { + let fixture = Fixture::new(); + fixture.stale(0); + fixture.stale(1); + let hash = fixture.hash(); + let calls = Cell::new(0); + let result = recover_scope( + &fixture.journal(), + &fixture.scope(), + &fixture.0, + &hash, + || { + calls.set(calls.get() + 1); + if calls.get() == 4 { + Err(refused()) + } else { + Ok(()) + } + }, + ); + assert!(result.is_err()); + assert!(!path(&fixture.0, 0).exists()); + assert!(path(&fixture.0, 1).exists()); + let selected = inspect_scope(&fixture.journal(), &fixture.scope(), &fixture.0).unwrap(); + assert_eq!(selected["sha256"], hash); + assert_eq!(selected["remaining"], 1); + assert_eq!(selected["resumable"], true); + assert_eq!(fixture.recover(&hash).unwrap()["removed"], 1); +} + +#[test] +fn owner_graph_boot_or_volume_proof_drift_refuses_before_any_effect() { + let fixture = Fixture::new(); + fixture.stale(0); + let hash = fixture.hash(); + let selected = fixture.selection(); + let changed = Scope { + quiescence_sha256: "b".repeat(64), + ..fixture.scope() + }; + assert!(recover_scope(&fixture.journal(), &changed, &fixture.0, &hash, || Ok(())).is_err()); + assert!( + recover_scope( + &fixture.journal(), + &fixture.scope(), + &fixture.0, + &hash, + || Err(refused()) + ) + .is_err() + ); + assert_eq!(fixture.selection(), selected); + assert!(!fixture.journal().exists()); +} + +#[test] +fn live_listener_and_foreign_replacement_are_never_removed() { + let fixture = Fixture::new(); + let listener = fixture.bind(0); + assert!(current(fixture.scope(), &fixture.0).is_err()); + drop(listener); + let hash = fixture.hash(); + fs::rename(path(&fixture.0, 0), fixture.0.join("original.sock")).unwrap(); + fixture.stale(0); + assert!(fixture.recover(&hash).is_err()); + assert!(path(&fixture.0, 0).exists()); + fs::remove_file(path(&fixture.0, 0)).unwrap(); + fs::write(path(&fixture.0, 0), b"foreign").unwrap(); + assert!(fixture.recover(&hash).is_err()); + assert_eq!(fs::read(path(&fixture.0, 0)).unwrap(), b"foreign"); +} + +#[test] +fn an_unselected_socket_or_pending_journal_blocks_retry() { + let fixture = Fixture::new(); + fixture.stale(0); + let hash = fixture.hash(); + state::private_directory(&fixture.journal()).unwrap(); + state::write( + &fixture.journal().join(format!("{hash}.json")), + &fixture.selection(), + ) + .unwrap(); + fixture.stale(1); + assert!(fixture.recover(&hash).is_err()); + assert!(path(&fixture.0, 0).exists()); + assert!(path(&fixture.0, 1).exists()); + fs::remove_file(path(&fixture.0, 1)).unwrap(); + let pending = fixture + .journal() + .join(format!("{}.pending", "c".repeat(64))); + fs::write(&pending, b"uncommitted").unwrap(); + assert!(fixture.recover(&hash).is_err()); + assert_eq!(fs::read(pending).unwrap(), b"uncommitted"); + assert!(path(&fixture.0, 0).exists()); +} + +#[test] +fn malformed_selection_and_stopped_pool_schema_cannot_authorize_live_recovery() { + let fixture = Fixture::new(); + fixture.stale(0); + let selected = fixture.selection(); + for sockets in [ + vec![selected.sockets[0].clone(), selected.sockets[0].clone()], + vec![], + ] { + assert!( + matching( + &Selection { + sockets, + ..selected.clone() + }, + &fixture.scope(), + &fixture.0 + ) + .is_err() + ); + } + assert!( + serde_json::from_value::( + json!({"version":1,"owner":"a","boot":"b","sockets":[]}) + ) + .is_err() + ); + assert!(fixture.recover("not-a-sha256").is_err()); + assert!(fixture.recover(&"0".repeat(64)).is_err()); + assert!(path(&fixture.0, 0).exists()); +} diff --git a/packages/runtime-core/src/provider/dependency_socket/recovery.rs b/packages/runtime-core/src/provider/dependency_socket/recovery.rs index 9aa655be4..3b0ff3df8 100644 --- a/packages/runtime-core/src/provider/dependency_socket/recovery.rs +++ b/packages/runtime-core/src/provider/dependency_socket/recovery.rs @@ -26,12 +26,12 @@ fn refused() -> CandidateError { ) } -fn digest(value: &T) -> Result { +pub(super) fn digest(value: &T) -> Result { let bytes = serde_json::to_vec(value).map_err(|_| refused())?; Ok(format!("{:x}", Sha256::digest(bytes))) } -fn hex(value: &str) -> bool { +pub(super) fn hex(value: &str) -> bool { value.len() == 64 && value .bytes() @@ -40,8 +40,8 @@ fn hex(value: &str) -> bool { #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] -struct Socket { - slot: u8, +pub(super) struct Socket { + pub(super) slot: u8, device: u64, inode: u64, } @@ -112,11 +112,11 @@ fn stopped(candidate: &Candidate) -> Result<(Selection, PathBuf), CandidateError )) } -fn path(directory: &Path, slot: u8) -> PathBuf { +pub(super) fn path(directory: &Path, slot: u8) -> PathBuf { directory.join(format!("dependency-{slot:02}.sock")) } -fn observed(directory: &Path, slot: u8) -> Result, CandidateError> { +pub(super) fn observed(directory: &Path, slot: u8) -> Result, CandidateError> { let target = path(directory, slot); let metadata = match fs::symlink_metadata(&target) { Ok(metadata) => metadata, diff --git a/packages/runtime-core/src/provider/graph/bridges/cleanup.rs b/packages/runtime-core/src/provider/graph/bridges/cleanup.rs index 17779815f..0b734c03c 100644 --- a/packages/runtime-core/src/provider/graph/bridges/cleanup.rs +++ b/packages/runtime-core/src/provider/graph/bridges/cleanup.rs @@ -51,6 +51,18 @@ fn strict_store(candidate: &Candidate, engine: &Engine<'_>) -> Result, +) -> Result<(), CandidateError> { + if !strict_store(candidate, engine)?.slots.is_empty() { + return Err(invalid()); + } + Ok(()) +} + fn validate_selection( engine: &Engine<'_>, receipt: &Receipt, diff --git a/packages/runtime-core/src/provider/graph/foreground/transport.rs b/packages/runtime-core/src/provider/graph/foreground/transport.rs index d92072cf9..fadb1ee40 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport.rs @@ -772,6 +772,9 @@ impl Publication { Self::bind_mode(candidate, run, true) } fn bind_mode(candidate: &Candidate, run: &str, retired: bool) -> Result { + // Publication precedes the Engine lease. Recovery must exclude new runs + // at this boundary, rather than relying on a later provider lock alone. + let gate = super::super::publication_gate::Guard::acquire(candidate)?; super::super::absent_publication_cleanup::publication_allowed(candidate, run, retired)?; let root = root(candidate, run)?; let lock = if retired { @@ -793,6 +796,7 @@ impl Publication { _ => return Err(refused()), } } + gate.verify(candidate)?; let listener = UnixListener::bind(root.join("control.sock")).map_err(|_| refused())?; fs::set_permissions(root.join("control.sock"), fs::Permissions::from_mode(0o600)) .map_err(|_| refused())?; @@ -828,6 +832,7 @@ impl Publication { device_rebind: None, }; pin.verify()?; + gate.verify(candidate)?; Ok(Self { listener, pin, diff --git a/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs b/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs index b81f11ee3..ba48e5f24 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs @@ -1,6 +1,43 @@ use super::*; use serde_json::json; +#[test] +fn pool_gate_excludes_publication_before_engine_for_every_run() { + let fixture = super::super::super::tests::Fixture::new(); + let candidate = Candidate::discover(&fixture.0).unwrap(); + let gate = super::super::super::publication_gate::Guard::acquire(&candidate).unwrap(); + for run in ["a".repeat(32), "b".repeat(32)] { + assert!(Publication::bind(&candidate, &run).is_err()); + assert!(!root(&candidate, &run).unwrap().exists()); + } + gate.verify(&candidate).unwrap(); + drop(gate); + let mut first = Publication::bind(&candidate, &"a".repeat(32)).unwrap(); + // A publisher keeps its own run lock, not the pool gate for its lifetime. + let mut second = Publication::bind(&candidate, &"b".repeat(32)).unwrap(); + first.finish().unwrap(); + second.finish().unwrap(); + for run in ["a".repeat(32), "b".repeat(32)] { + fs::remove_dir_all(root(&candidate, &run).unwrap()).unwrap(); + } +} + +#[test] +fn pool_gate_refuses_a_replaced_lock_path() { + let fixture = super::super::super::tests::Fixture::new(); + let candidate = Candidate::discover(&fixture.0).unwrap(); + let gate = super::super::super::publication_gate::Guard::acquire(&candidate).unwrap(); + let directory = candidate.state_root.join("run/graph-publication-gate"); + fs::rename( + directory.join("operation.lock"), + directory.join("original.lock"), + ) + .unwrap(); + let _replacement = state::Lock::acquire(&directory).unwrap(); + assert!(gate.verify(&candidate).is_err()); + assert!(directory.join("original.lock").exists()); +} + #[cfg(target_os = "macos")] fn abandoned_publisher() -> ( super::super::super::tests::Fixture, diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index 1001b3330..5edf87f59 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -22,6 +22,10 @@ mod dependency_slots; #[cfg(target_os = "macos")] mod host_pin_recovery; #[cfg(target_os = "macos")] +pub(crate) mod publication_gate; +#[cfg(target_os = "macos")] +pub(crate) mod quiescent_dependency_recovery; +#[cfg(target_os = "macos")] pub use absent_publication_cleanup::{ inspect as inspect_absent_publication_cleanup, recover as recover_absent_publication_cleanup, }; diff --git a/packages/runtime-core/src/provider/graph/publication_gate.rs b/packages/runtime-core/src/provider/graph/publication_gate.rs new file mode 100644 index 000000000..a43db0dcc --- /dev/null +++ b/packages/runtime-core/src/provider/graph/publication_gate.rs @@ -0,0 +1,50 @@ +//! Serialize publication effects with explicit pool-wide quiescence recovery. +//! Publishers release this gate after binding; their ordinary run lock remains held. +//! Recovery retains it before taking run locks and the provider engine lease. +use crate::{Candidate, CandidateError, provider::state}; +use std::{fs, os::unix::fs::MetadataExt, path::PathBuf}; + +pub(crate) struct Guard { + root: PathBuf, + directory: (u64, u64), + lock: state::Lock, +} + +impl Guard { + pub(crate) fn acquire(candidate: &Candidate) -> Result { + let root = candidate.state_root.join("run/graph-publication-gate"); + let lock = state::Lock::acquire(&root)?; + let metadata = fs::symlink_metadata(&root).map_err(state::io)?; + let guard = Self { + root, + directory: (metadata.dev(), metadata.ino()), + lock, + }; + guard.verify(candidate)?; + Ok(guard) + } + + pub(crate) fn verify(&self, candidate: &Candidate) -> Result<(), CandidateError> { + let root = candidate.state_root.join("run/graph-publication-gate"); + state::check_private_directory(&root)?; + let directory = fs::symlink_metadata(&root).map_err(state::io)?; + let lock = fs::symlink_metadata(root.join("operation.lock")).map_err(state::io)?; + if self.root != root + || !directory.is_dir() + || directory.file_type().is_symlink() + || (directory.dev(), directory.ino()) != self.directory + || !lock.is_file() + || lock.file_type().is_symlink() + || lock.nlink() != 1 + || lock.uid() != unsafe { libc::geteuid() } + || lock.mode() & 0o077 != 0 + || (lock.dev(), lock.ino()) != self.lock.identity()? + { + return Err(CandidateError::new( + "graph_publication_gate", + "Graph publication gate changed; recovery and publication were refused.", + )); + } + Ok(()) + } +} diff --git a/packages/runtime-core/src/provider/graph/quiescent_dependency_recovery.rs b/packages/runtime-core/src/provider/graph/quiescent_dependency_recovery.rs new file mode 100644 index 000000000..75f3cb627 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/quiescent_dependency_recovery.rs @@ -0,0 +1,453 @@ +//! Exact pool quiescence proof for explicit legacy dependency-socket recovery. +//! This grants no graph cleanup or data deletion authority. In particular, a +//! private unrecorded socket inode does not prove who originally created it. +use super::{ + Candidate, CandidateError, Engine, Kind, Method, Receipt, directory, host_pin_recovery, + inspect_resource, load_at, publication_gate, restore, state, +}; +use crate::provider::{lifecycle, state::Owner}; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + fs, + os::unix::fs::MetadataExt, + path::{Path, PathBuf}, +}; + +const MAX_GRAPHS: usize = 64; +const MAX_GRAPH_FILES: usize = 256; +const MAX_STATE_BYTES: u64 = 2 * 1024 * 1024; + +fn refused() -> CandidateError { + CandidateError::new( + "dependency_socket_recovery", + "Pool quiescence or selected runtime identity changed; dependency sockets and data were retained.", + ) +} + +fn id(path: &Path) -> Result<(u64, u64), CandidateError> { + let metadata = fs::symlink_metadata(path).map_err(|_| refused())?; + Ok((metadata.dev(), metadata.ino())) +} + +fn absent(path: &Path) -> Result { + match fs::symlink_metadata(path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true), + Ok(_) => Ok(false), + Err(_) => Err(refused()), + } +} + +type GraphInventory = (Option<(u64, u64)>, Vec); + +fn graph_runs(candidate: &Candidate) -> Result { + let root = candidate.state_root.join("run/graphs"); + if absent(&root)? { + return Ok((None, Vec::new())); + } + state::check_private_directory(&root).map_err(|_| refused())?; + let identity = id(&root)?; + let entries = fs::read_dir(&root) + .map_err(|_| refused())? + .take(MAX_GRAPHS + 1) + .collect::, _>>() + .map_err(|_| refused())?; + if entries.len() > MAX_GRAPHS { + return Err(refused()); + } + let mut runs = Vec::with_capacity(entries.len()); + for entry in entries { + let run = entry.file_name().into_string().map_err(|_| refused())?; + if !super::hex(&run, 32) || directory(candidate, &run)? != entry.path() { + return Err(refused()); + } + state::check_private_directory(&entry.path()).map_err(|_| refused())?; + runs.push(run); + } + runs.sort(); + if runs.windows(2).any(|pair| pair[0] == pair[1]) { + return Err(refused()); + } + Ok((Some(identity), runs)) +} + +struct Foreground { + root: PathBuf, + directory: (u64, u64), + lock: state::Lock, +} + +impl Foreground { + fn acquire(candidate: &Candidate, run: &str) -> Result { + let root = super::foreground::transport::root(candidate, run)?; + let lock = if absent(&root)? { + state::Lock::acquire(&root)? + } else { + state::check_private_directory(&root).map_err(|_| refused())?; + state::Lock::acquire_existing(&root)? + }; + let result = Self { + directory: id(&root)?, + root, + lock, + }; + result.verify()?; + Ok(result) + } + + fn verify(&self) -> Result<(), CandidateError> { + state::check_private_directory(&self.root).map_err(|_| refused())?; + if id(&self.root)? != self.directory { + return Err(refused()); + } + host_pin_recovery::exact_lock_path(&self.root, &self.lock).map_err(|_| refused())?; + let entries = fs::read_dir(&self.root) + .map_err(|_| refused())? + .take(2) + .collect::, _>>() + .map_err(|_| refused())?; + if entries.len() != 1 || entries[0].file_name().to_str() != Some("operation.lock") { + return Err(refused()); + } + Ok(()) + } +} + +#[derive(Clone, PartialEq, Eq, Serialize)] +struct GraphProof { + run: String, + graph_root: (u64, u64), + state_file: (u64, u64), + state_sha256: String, + foreground_root: (u64, u64), + retained_volumes: BTreeMap, +} + +#[derive(Clone, PartialEq, Eq, Serialize)] +struct Selection { + version: u8, + checkout: PathBuf, + owner_sha256: String, + owner_file: (u64, u64), + owner_root: (u64, u64), + owner_token: String, + guest_boot: String, + host_boot_micros: u64, + home: (u64, u64), + graph_directory: Option<(u64, u64)>, + dependency_assignments: Option<(u64, u64)>, + graphs: Vec, +} + +fn no_pending(root: &Path) -> Result<(), CandidateError> { + let entries = fs::read_dir(root) + .map_err(|_| refused())? + .take(MAX_GRAPH_FILES + 1) + .collect::, _>>() + .map_err(|_| refused())?; + if entries.len() > MAX_GRAPH_FILES + || entries.iter().any(|entry| { + entry + .file_name() + .to_str() + .is_none_or(|name| name.ends_with(".pending")) + }) + { + return Err(refused()); + } + Ok(()) +} + +fn no_dependency_assignments(candidate: &Candidate) -> Result, CandidateError> { + let root = candidate.state_root.join("run/dependency-assignments"); + if absent(&root)? { + return Ok(None); + } + state::check_private_directory(&root).map_err(|_| refused())?; + if fs::read_dir(&root).map_err(|_| refused())?.next().is_some() { + return Err(refused()); + } + Ok(Some(id(&root)?)) +} + +fn no_guest_containers(engine: &Engine<'_>) -> Result<(), CandidateError> { + let inventory = engine.request( + Method::GET, + "/v1.53/containers/json?all=true&limit=129", + None, + )?; + let entries = inventory.as_array().ok_or_else(refused)?; + // A nonempty list includes either a still-live consumer or an unrecorded + // stopped object whose mounts cannot be attributed to this selection. + if !entries.is_empty() { + return Err(refused()); + } + Ok(()) +} + +fn no_compute(engine: &Engine<'_>, receipt: &Receipt) -> Result<(), CandidateError> { + for resource in receipt.resources.values() { + if resource.kind == Kind::Volume { + continue; + } + if resource.phase != "absent" || inspect_resource(engine, receipt, resource)?.is_some() { + return Err(refused()); + } + let suffix = if resource.kind == Kind::Container { + "/json" + } else { + "" + }; + let path = format!( + "/v1.53/{}/{}{}", + resource.kind.collection(), + resource.name, + suffix + ); + match engine.request(Method::GET, &path, None) { + Err(error) if error.code == "engine_not_found" => {} + _ => return Err(refused()), + } + } + Ok(()) +} + +fn stopped_startup(receipt: &Receipt) -> Result<&super::startup::Startup, CandidateError> { + if receipt.phase != "stopped-data-retained" || receipt.relay_cleanup.is_some() { + return Err(refused()); + } + receipt.relay_startup.as_ref().ok_or_else(refused) +} + +fn graph_proof( + candidate: &Candidate, + engine: &Engine<'_>, + run: &str, + foreground: &Foreground, +) -> Result { + foreground.verify()?; + let root = directory(candidate, run)?; + no_pending(&root)?; + let (receipt, _) = load_at(root.clone(), run, engine.guest().incarnation())?; + let startup = stopped_startup(&receipt)?; + if !absent(&startup.control_root)? { + return Err(refused()); + } + let state_path = root.join("state.json"); + let state_bytes = host_pin_recovery::read_raw(&state_path, MAX_STATE_BYTES)?; + let raw_receipt: Receipt = serde_json::from_slice(&state_bytes).map_err(|_| refused())?; + if serde_json::to_value(&raw_receipt).map_err(|_| refused())? + != serde_json::to_value(&receipt).map_err(|_| refused())? + { + return Err(refused()); + } + no_compute(engine, &receipt)?; + let retained_volumes = restore::observed_volumes(engine, &receipt)?; + Ok(GraphProof { + run: run.into(), + graph_root: id(&root)?, + state_file: id(&state_path)?, + state_sha256: format!("{:x}", Sha256::digest(state_bytes)), + foreground_root: foreground.directory, + retained_volumes, + }) +} + +fn observe( + candidate: &Candidate, + engine: &Engine<'_>, + foreground: &[Foreground], +) -> Result<(Owner, Selection), CandidateError> { + lifecycle::host_filesystem::no_auxiliary_update(candidate)?; + let owner = Owner::load(candidate)?; + if owner.phase != "running" + || owner.token != engine.guest().incarnation() + || owner.guest_boot_id.as_deref() != Some(engine.guest().boot_id()) + || owner.dependency_sockets.is_none() + { + return Err(refused()); + } + host_pin_recovery::verify_guest_identity(engine)?; + let owner_root = candidate.state_root.join("run/smolvm"); + let owner_path = owner_root.join("owner.json"); + let owner_bytes = host_pin_recovery::read_raw(&owner_path, 1024 * 1024)?; + if serde_json::from_slice::(&owner_bytes).map_err(|_| refused())? != owner { + return Err(refused()); + } + let home = owner_root.join("home"); + state::check_private_directory(&home).map_err(|_| refused())?; + let home_id = id(&home)?; + let alias = fs::metadata(&owner.short_home).map_err(|_| refused())?; + if (alias.dev(), alias.ino()) != home_id { + return Err(refused()); + } + let (graph_directory, runs) = graph_runs(candidate)?; + if runs.len() != foreground.len() { + return Err(refused()); + } + no_guest_containers(engine)?; + super::bridges::cleanup::require_quiescent(candidate, engine)?; + let mut graphs = Vec::with_capacity(runs.len()); + for (run, guard) in runs.iter().zip(foreground) { + graphs.push(graph_proof(candidate, engine, run, guard)?); + } + let selected = Selection { + version: 1, + checkout: candidate.checkout.clone(), + owner_sha256: format!("{:x}", Sha256::digest(owner_bytes)), + owner_file: id(&owner_path)?, + owner_root: id(&owner_root)?, + owner_token: owner.token.clone(), + guest_boot: engine.guest().boot_id().into(), + host_boot_micros: lifecycle::host_filesystem::host_boot_micros()?, + home: home_id, + graph_directory, + dependency_assignments: no_dependency_assignments(candidate)?, + graphs, + }; + Ok((owner, selected)) +} + +/// Holds the publication gate, every selected foreground lock, and then the +/// provider Engine lease. Rechecking this exact observation is required before +/// each socket unlink and after the last unlink. +pub(crate) struct Guard<'a> { + candidate: &'a Candidate, + gate: publication_gate::Guard, + foreground: Vec, + engine: Engine<'a>, + owner: Owner, + selected: Selection, + sha256: String, +} + +impl<'a> Guard<'a> { + pub(crate) fn acquire(candidate: &'a Candidate) -> Result { + let gate = publication_gate::Guard::acquire(candidate)?; + let (_, runs) = graph_runs(candidate)?; + let mut foreground = Vec::with_capacity(runs.len()); + for run in &runs { + foreground.push(Foreground::acquire(candidate, run)?); + } + gate.verify(candidate)?; + let engine = Engine::connect_cleanup_wait(candidate)?; + let (owner, selected) = observe(candidate, &engine, &foreground)?; + let bytes = serde_json::to_vec(&selected).map_err(|_| refused())?; + let sha256 = format!("{:x}", Sha256::digest(bytes)); + let guard = Self { + candidate, + gate, + foreground, + engine, + owner, + selected, + sha256, + }; + guard.verify(candidate)?; + Ok(guard) + } + + pub(crate) fn verify(&self, candidate: &Candidate) -> Result<(), CandidateError> { + if self.candidate.state_root != candidate.state_root + || self.candidate.checkout != candidate.checkout + { + return Err(refused()); + } + self.gate.verify(candidate)?; + for guard in &self.foreground { + guard.verify()?; + } + self.engine.guest().verify()?; + let (owner, selected) = observe(candidate, &self.engine, &self.foreground)?; + if owner != self.owner || selected != self.selected { + return Err(refused()); + } + self.gate.verify(candidate)?; + Ok(()) + } + + pub(crate) fn sha256(&self) -> &str { + &self.sha256 + } + + pub(crate) fn owner(&self) -> &Owner { + &self.owner + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn graph_inventory_accepts_only_absent_or_bounded_private_run_directories() { + let fixture = super::super::tests::Fixture::new(); + let candidate = Candidate::discover(&fixture.0).unwrap(); + assert_eq!(graph_runs(&candidate).unwrap(), (None, Vec::new())); + let graphs = candidate.state_root.join("run/graphs"); + state::private_directory(&graphs).unwrap(); + assert_eq!(graph_runs(&candidate).unwrap().1, Vec::::new()); + fs::write(graphs.join("unknown"), b"not a graph").unwrap(); + assert!(graph_runs(&candidate).is_err()); + fs::remove_file(graphs.join("unknown")).unwrap(); + state::private_directory(&graphs.join("a".repeat(32))).unwrap(); + assert_eq!(graph_runs(&candidate).unwrap().1, vec!["a".repeat(32)]); + } + + #[test] + fn pending_and_active_graphs_never_satisfy_stopped_policy() { + let fixture = super::super::tests::Fixture::new(); + let root = fixture.0.join("graph"); + state::private_directory(&root).unwrap(); + assert!(no_pending(&root).is_ok()); + fs::write(root.join("state.pending"), b"interrupted").unwrap(); + assert!(no_pending(&root).is_err()); + fs::remove_file(root.join("state.pending")).unwrap(); + fs::write(root.join("unknown.pending"), b"interrupted").unwrap(); + assert!(no_pending(&root).is_err()); + + let mut receipt: Receipt = serde_json::from_value(json!({ + "version":1,"run":"a".repeat(32),"owner":"b".repeat(32), + "namespace":"c".repeat(64),"plan_id":"d".repeat(64), + "phase":"ready-observed","readiness":{},"resources":{}, + "relay_startup":{"control_only":true,"guest_root":null, + "control_root":"/private/missing","artifact":"e".repeat(64),"services":{}} + })) + .unwrap(); + assert!(stopped_startup(&receipt).is_err()); + receipt.phase = "stopped-data-retained".into(); + assert!(stopped_startup(&receipt).is_ok()); + receipt.relay_startup = None; + assert!(stopped_startup(&receipt).is_err()); + } + + #[test] + fn dependency_assignment_directory_must_be_empty_and_private() { + let fixture = super::super::tests::Fixture::new(); + let candidate = Candidate::discover(&fixture.0).unwrap(); + assert_eq!(no_dependency_assignments(&candidate).unwrap(), None); + let assignments = candidate.state_root.join("run/dependency-assignments"); + state::private_directory(&assignments).unwrap(); + assert!(no_dependency_assignments(&candidate).unwrap().is_some()); + fs::write(assignments.join("stale.json"), b"selected").unwrap(); + assert!(no_dependency_assignments(&candidate).is_err()); + } + + #[test] + fn foreground_reservation_refuses_replaced_lock_path() { + let fixture = super::super::tests::Fixture::new(); + let candidate = Candidate::discover(&fixture.0).unwrap(); + let run = "a".repeat(32); + let guard = Foreground::acquire(&candidate, &run).unwrap(); + guard.verify().unwrap(); + let root = guard.root.clone(); + fs::rename(root.join("operation.lock"), root.join("old.lock")).unwrap(); + let replacement = state::Lock::acquire(&root).unwrap(); + assert!(guard.verify().is_err()); + drop(replacement); + drop(guard); + fs::remove_dir_all(root).unwrap(); + } +} diff --git a/packages/runtime-core/src/provider/mod.rs b/packages/runtime-core/src/provider/mod.rs index d1203b751..d378e5a56 100644 --- a/packages/runtime-core/src/provider/mod.rs +++ b/packages/runtime-core/src/provider/mod.rs @@ -6,6 +6,7 @@ mod bridge; mod config_audit; mod dependency_socket; pub use bridge::BridgeIntent; +pub use dependency_socket::quiescent_recovery as quiescent_dependency_socket_recovery; pub use dependency_socket::recovery as dependency_socket_recovery; pub use dependency_socket::{ DependencySocketIntent, DependencySocketObservation, DependencySocketPath, diff --git a/packages/runtime-core/tests/quiescent_dependency_recovery_cli_contract.rs b/packages/runtime-core/tests/quiescent_dependency_recovery_cli_contract.rs new file mode 100644 index 000000000..fa1a831ee --- /dev/null +++ b/packages/runtime-core/tests/quiescent_dependency_recovery_cli_contract.rs @@ -0,0 +1,60 @@ +//! An explicit live recovery selection is required before provider access. +use serde_json::Value; +use std::{fs, path::Path, process::Command}; + +#[test] +fn quiescent_recovery_rejects_missing_duplicate_and_invalid_selection() { + let checkout = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .unwrap(); + let home = std::env::temp_dir().join(format!("hack-quiescent-cli-{}", std::process::id())); + fs::create_dir(&home).unwrap(); + let cases = [ + vec!["runtime", "recover-quiescent-dependency-sockets", "--json"], + vec![ + "runtime", + "recover-quiescent-dependency-sockets", + "--expect-sha256", + "invalid", + "--json", + ], + vec![ + "runtime", + "recover-quiescent-dependency-sockets", + "--expect-sha256", + "invalid", + "--json", + "--json", + ], + vec![ + "runtime", + "quiescent-dependency-socket-recovery", + "--json", + "--json", + ], + ]; + for arguments in cases { + let output = Command::new(env!("CARGO_BIN_EXE_hack-runtime-candidate")) + .arg("--candidate-root") + .arg(&checkout) + .args(&arguments) + .env_clear() + .env("PATH", "/nonexistent") + .env("HOME", &home) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2), "{arguments:?}"); + assert!(output.stdout.is_empty()); + let failure: Value = serde_json::from_slice(&output.stderr).unwrap(); + assert!( + matches!( + failure["code"].as_str(), + Some("unsupported_command" | "quiescent_dependency_socket_recovery") + ), + "{failure:?}" + ); + assert_eq!(fs::read_dir(&home).unwrap().count(), 0); + } + fs::remove_dir(home).unwrap(); +} From 7dc0811cd0179f40340e48c540699933dc099cf7 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 11:30:07 -0400 Subject: [PATCH 20/25] fix(runtime): preserve witnessed dependency cache scope on restore --- packages/runtime-core/README.md | 17 ++ .../runtime-core/src/provider/graph/config.rs | 5 +- .../src/provider/graph/dependency_cache.rs | 273 +++++++++++++++++- .../src/provider/graph/hostname_change.rs | 1 + .../runtime-core/src/provider/graph/mod.rs | 4 + .../src/provider/graph/restore.rs | 3 + .../runtime-core/src/provider/graph/source.rs | 113 +++++++- .../provider/graph/source_device_rebind.rs | 173 ++++++++++- .../runtime-core/src/provider/graph/tests.rs | 2 + 9 files changed, 580 insertions(+), 11 deletions(-) diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 9444e009b..0bb4599ae 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -853,3 +853,20 @@ there is no automatic prefix-based adoption or deletion. The ignored native synthetic-device fixture checks this same-run transition and later retained marker reads, but only an actual host-reboot application run can qualify the physical continuity claim. + +A selected source-device witness also preserves an existing dependency cache's +namespace when the Git common directory is on that same filesystem. The runtime +reconstructs the old scope hash from the unchanged common directory path/inode +and selected prior device number, and requires every retained cache scope to +match. Only the new attempt receives optional cache-scope continuity metadata; +the stopped receipt, original cache names/fingerprints, volumes and witness bytes +stay unchanged. All package inputs, image, command, environment, host mappings +and volume layouts are still freshly fingerprinted, with strict resource equality. +This is not a fallback for changed lockfiles or an unrelated repository. + +Later ordinary replays verify the immutable witness's raw hash and run/owner/share +linkage, the current common directory path/device/inode and retained scopes. +Missing, changed or pending provenance and replaced Git metadata refuse. A second +device transition requires a separately supported explicit recovery; the runtime +does not silently extend this projection. Older candidate binaries may refuse the +new optional source metadata, so a downgrade must be qualified separately. diff --git a/packages/runtime-core/src/provider/graph/config.rs b/packages/runtime-core/src/provider/graph/config.rs index df8b3747d..c84f57fcb 100644 --- a/packages/runtime-core/src/provider/graph/config.rs +++ b/packages/runtime-core/src/provider/graph/config.rs @@ -155,7 +155,10 @@ pub(super) fn prepare_delivery( "Dependency caches require an explicitly verified source publication.", ) })?; - let scope = dependency_cache::scope(&plan.source)?; + let scope = source.binding.cache_scope.as_ref().map_or_else( + || dependency_cache::scope(&plan.source), + |scope| scope.scope(&plan.source), + )?; dependency_cache::resolve( plan, source.current_manifest.as_ref().unwrap_or(&source.manifest), diff --git a/packages/runtime-core/src/provider/graph/dependency_cache.rs b/packages/runtime-core/src/provider/graph/dependency_cache.rs index a4af6ad93..ca6a44b63 100644 --- a/packages/runtime-core/src/provider/graph/dependency_cache.rs +++ b/packages/runtime-core/src/provider/graph/dependency_cache.rs @@ -257,7 +257,7 @@ fn metadata_text(path: &Path) -> Result { Ok(value.to_owned()) } /// Uses bounded Git metadata only; never invokes Git or reads configuration/credentials. -pub fn scope(project: &Path) -> Result { +fn scope_root(project: &Path) -> Result { let project = owned_directory(project)?; let dotgit = project.join(".git"); let metadata = match fs::symlink_metadata(&dotgit) { @@ -285,13 +285,148 @@ pub fn scope(project: &Path) -> Result { } _ => return Err(refused()), }; - let metadata = fs::metadata(&common).map_err(|_| refused())?; - hash(&( - "hack-dependency-cache-scope-v1", + let directory = fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_DIRECTORY) + .open(&common) + .map_err(|_| refused())?; + let metadata = directory.metadata().map_err(|_| refused())?; + let observed = fs::symlink_metadata(&common).map_err(|_| refused())?; + if !metadata.is_dir() + || !observed.is_dir() + || metadata.dev() != observed.dev() + || metadata.ino() != observed.ino() + { + return Err(refused()); + } + Ok(ScopeRoot { common, - metadata.dev(), - metadata.ino(), - )) + device: metadata.dev(), + inode: metadata.ino(), + }) +} + +struct ScopeRoot { + common: PathBuf, + device: u64, + inode: u64, +} +impl ScopeRoot { + fn hash(&self, device: u64) -> Result { + hash(&( + "hack-dependency-cache-scope-v1", + &self.common, + device, + self.inode, + )) + } +} + +pub fn scope(project: &Path) -> Result { + let root = scope_root(project)?; + root.hash(root.device) +} + +/// Continuity of an existing cache namespace after a selected device rebind. +/// This records no package inputs and grants no volume creation or adoption. +/// Every replay still recomputes the full fingerprint and compares exact resources. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ReplayScope { + common: PathBuf, + device: u64, + inode: u64, + original_device: u64, + origin_sha256: String, +} +impl ReplayScope { + pub(super) fn valid(&self) -> bool { + self.common.is_absolute() + && self.common.as_os_str().len() <= 4096 + && self.common.to_str().is_some_and(|path| { + !path.chars().any(char::is_control) + && self + .common + .components() + .all(|c| matches!(c, Component::RootDir | Component::Normal(_))) + }) + && self.inode != 0 + && hex(&self.origin_sha256) + } + + pub(super) fn scope(&self, project: &Path) -> Result { + let root = scope_root(project)?; + if !self.valid() + || root.common != self.common + || root.device != self.device + || root.inode != self.inode + { + return Err(refused()); + } + root.hash(self.original_device) + } + + /// Called only by an already selected source-device witness. Reconstruct the + /// old hash using the same Git common path/inode and require every retained + /// cache to match it. Git metadata on a different filesystem is not projected. + #[cfg(any(target_os = "macos", test))] + pub(super) fn project( + old: &crate::provider::ProjectShareIntent, + current: &crate::provider::ProjectShareIntent, + previous: Option<&Self>, + retained: &[&str], + origin_sha256: &str, + ) -> Result, CandidateError> { + if retained.is_empty() { + return if previous.is_none() { + Ok(None) + } else { + Err(refused()) + }; + } + if !hex(origin_sha256) + || previous.is_some() + || old.device == current.device + || old.project != current.project + || old.inode != current.inode + { + return Err(refused()); + } + let root = scope_root(¤t.project)?; + if root.device != current.device { + return Err(refused()); + } + let original_device = old.device; + let expected = root.hash(original_device)?; + if retained.iter().any(|scope| *scope != expected) { + return Err(refused()); + } + Ok(Some(Self { + common: root.common, + device: root.device, + inode: root.inode, + original_device, + origin_sha256: origin_sha256.into(), + })) + } + + #[cfg(target_os = "macos")] + pub(super) fn matches_origin( + &self, + sha256: &str, + old: &crate::provider::ProjectShareIntent, + current: &crate::provider::ProjectShareIntent, + ) -> bool { + self.valid() + && self.origin_sha256 == sha256 + && self.original_device == old.device + && self.device == current.device + && old.device != current.device + && old.project == current.project + && old.inode == current.inode + && old.guest_path == current.guest_path + && old.unfiltered_source == current.unfiltered_source + } } #[cfg(test)] @@ -364,6 +499,130 @@ mod tests { } .unwrap(); } + + #[test] + fn selected_device_replay_keeps_exact_scope_and_fingerprint_inputs() { + let (source, _home, plan, manifest) = prepared(); + let root = scope_root(&source.0).unwrap(); + let current = crate::provider::ProjectShareIntent::approve(&source.0, true).unwrap(); + let mut old = current.clone(); + old.device += 7; + let retained = root.hash(old.device).unwrap(); + let projected = ReplayScope::project(&old, ¤t, None, &[&retained], &"e".repeat(64)) + .unwrap() + .unwrap(); + assert_eq!(projected.scope(&source.0).unwrap(), retained); + assert_ne!(scope(&source.0).unwrap(), retained); + let persisted: ReplayScope = + serde_json::from_slice(&serde_json::to_vec(&projected).unwrap()).unwrap(); + assert_eq!(persisted.scope(&source.0).unwrap(), retained); + let expected = resolve(&plan, &manifest, &retained).unwrap(); + assert_eq!( + resolve(&plan, &manifest, &persisted.scope(&source.0).unwrap()).unwrap(), + expected + ); + + let mut changed = manifest.clone(); + changed + .entries + .iter_mut() + .find(|entry| entry.path == "bun.lock") + .unwrap() + .sha256 = Some("f".repeat(64)); + resign(&mut changed); + assert_ne!(resolve(&plan, &changed, &retained).unwrap(), expected); + for field in ["environment", "extra_hosts", "entrypoint"] { + let mut executable = executable(&plan); + let service = executable.get_mut("deps").unwrap(); + match field { + "environment" => service.environment.push("NEW=public".into()), + "extra_hosts" => { + service + .extra_hosts + .insert("changed.example".into(), "host-gateway".into()); + } + "entrypoint" => service.entrypoint = Some(vec!["/changed".into()]), + _ => unreachable!(), + } + assert_ne!( + super::resolve(&plan, &manifest, &retained, &executable).unwrap(), + expected + ); + } + assert!( + ReplayScope::project(&old, ¤t, None, &[&"a".repeat(64)], &"e".repeat(64)) + .is_err() + ); + let mut other_filesystem = current.clone(); + other_filesystem.device += 1; + assert!( + ReplayScope::project(&old, &other_filesystem, None, &[&retained], &"e".repeat(64)) + .is_err() + ); + } + + #[test] + fn replay_scope_refuses_replaced_git_root_and_a_second_device_change() { + let fixture = Fixture::new(); + fs::create_dir(fixture.0.join(".git")).unwrap(); + let root = scope_root(&fixture.0).unwrap(); + let current = crate::provider::ProjectShareIntent { + project: fixture.0.clone(), + guest_path: "/mnt/hack-projects/exact".into(), + device: root.device, + inode: fs::metadata(&fixture.0).unwrap().ino(), + unfiltered_source: true, + }; + let mut old = current.clone(); + old.device += 2; + let retained = root.hash(old.device).unwrap(); + let projected = ReplayScope::project(&old, ¤t, None, &[&retained], &"e".repeat(64)) + .unwrap() + .unwrap(); + let mut previous = projected.clone(); + previous.device = old.device + 1; + let mut intermediate = old.clone(); + intermediate.device = previous.device; + assert!( + ReplayScope::project( + &intermediate, + ¤t, + Some(&previous), + &[&retained], + &"e".repeat(64) + ) + .is_err() + ); + previous.inode += 1; + assert!( + ReplayScope::project( + &intermediate, + ¤t, + Some(&previous), + &[&retained], + &"e".repeat(64) + ) + .is_err() + ); + + fs::rename(fixture.0.join(".git"), fixture.0.join("original-git")).unwrap(); + fs::create_dir(fixture.0.join(".git")).unwrap(); + assert!(projected.scope(&fixture.0).is_err()); + assert!(ReplayScope::project(&old, ¤t, None, &[&retained], &"e".repeat(64)).is_err()); + assert!( + ReplayScope::project( + &old, + ¤t, + Some(&projected), + &[&retained], + &"e".repeat(64) + ) + .is_err() + ); + fs::remove_dir(fixture.0.join(".git")).unwrap(); + symlink(fixture.0.join("original-git"), fixture.0.join(".git")).unwrap(); + assert!(projected.scope(&fixture.0).is_err()); + } #[test] fn subpath_layout_changes_cache_identity_without_compose_digest_shortcut() { let (_source, _home, mut plan, manifest) = prepared(); diff --git a/packages/runtime-core/src/provider/graph/hostname_change.rs b/packages/runtime-core/src/provider/graph/hostname_change.rs index 7f8242089..45a96fcef 100644 --- a/packages/runtime-core/src/provider/graph/hostname_change.rs +++ b/packages/runtime-core/src/provider/graph/hostname_change.rs @@ -213,6 +213,7 @@ pub(super) mod tests { }; let binding = source::SourceBinding { shared: None, + cache_scope: None, shared_contract: Some( project::live_source::Contract::from_plan(&plan.plan, snapshot.receipt()).unwrap(), ), diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index 5edf87f59..c54b3fb3b 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -1765,6 +1765,7 @@ pub fn restart(candidate: &Candidate, options: RunOptions<'_>) -> Result Duration::from_secs(600) { return Err(error("graph_budget", "Invalid graph timeout.")); } + let project_path = options.project.project; let inputs = project::inputs::compile( candidate, options.project, @@ -1853,6 +1854,9 @@ pub fn restart(candidate: &Candidate, options: RunOptions<'_>) -> Result, #[serde(default, skip_serializing_if = "Option::is_none")] pub shared: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -30,6 +33,10 @@ impl SourceBinding { .all(|v| hex(v, 64)) && !(self.live.is_some() && self.shared.is_some()) && (self.shared_contract.is_none() || self.shared.is_some()) + && self + .cache_scope + .as_ref() + .is_none_or(|scope| self.shared.is_some() && scope.valid()) && self.live.as_ref().is_none_or(|live| live.workspace.valid()) && self .shared @@ -228,6 +235,7 @@ pub(super) fn prepare_mode( current_manifest: None, binding: SourceBinding { shared: Some(share.clone()), + cache_scope: None, shared_contract: Some(project::live_source::Contract::from_plan(plan, &manifest)?), live: None, revision: manifest.revision.clone(), @@ -308,6 +316,19 @@ pub(super) fn prepare_replay( shared: bool, non_secret_values: &BTreeMap, ) -> Result, CandidateError> { + #[cfg(target_os = "macos")] + super::source_device_rebind::verify_cache_scope_origin(engine.guest().candidate(), receipt)?; + #[cfg(not(target_os = "macos"))] + if receipt + .source + .as_ref() + .is_some_and(|binding| binding.cache_scope.is_some()) + { + return Err(error( + "graph_source_device_rebind", + "Cache device replay requires its native witness.", + )); + } if shared || receipt.source.as_ref().is_some_and(|s| s.shared.is_some()) { if !shared || live { return Err(error( @@ -323,7 +344,7 @@ pub(super) fn prepare_replay( revision, )?)); } - let source = prepare_mode( + let mut source = prepare_mode( engine.guest().candidate(), engine, &inputs.review.plan, @@ -331,6 +352,7 @@ pub(super) fn prepare_replay( false, true, )?; + inherit_cache_scope(&mut source, receipt, &inputs.review.plan.source)?; unchanged(&source, receipt)?; return Ok(source); } @@ -585,6 +607,7 @@ fn published_inputs( manifest: publication.manifest.clone(), binding: SourceBinding { shared: None, + cache_scope: None, shared_contract: None, live: None, revision: revision.into(), @@ -606,6 +629,36 @@ pub(super) fn unchanged(source: &Option, receipt: &Receipt) -> Result<() Ok(()) } +fn inherit_cache_scope( + source: &mut Option, + receipt: &Receipt, + project: &Path, +) -> Result<(), CandidateError> { + if let Some(scope) = receipt + .source + .as_ref() + .and_then(|binding| binding.cache_scope.as_ref()) + { + scope.scope(project)?; + source + .as_mut() + .ok_or_else(|| error("graph_source_changed", "Missing replay source."))? + .binding + .cache_scope = Some(scope.clone()); + } + Ok(()) +} + +pub(super) fn verify_cache_scope( + source: Option<&Inputs>, + project: &Path, +) -> Result<(), CandidateError> { + if let Some(scope) = source.and_then(|source| source.binding.cache_scope.as_ref()) { + scope.scope(project)?; + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -639,6 +692,7 @@ mod tests { }; let binding = SourceBinding { shared: None, + cache_scope: None, shared_contract: None, revision: baseline.receipt().revision.clone(), archive_sha256: publication.archive_sha256.clone(), @@ -778,6 +832,7 @@ mod tests { probes: BTreeMap::new(), source: Some(SourceBinding { shared: None, + cache_scope: None, shared_contract: None, revision: manifest.revision, archive_sha256: "d".repeat(64), @@ -896,6 +951,7 @@ mod tests { manifest: manifest.clone(), binding: SourceBinding { shared: None, + cache_scope: None, shared_contract: None, live: None, revision: manifest.revision.clone(), @@ -973,6 +1029,7 @@ mod tests { assert_eq!(serde_json::to_value(&receipt).unwrap(), value); let binding = SourceBinding { shared: None, + cache_scope: None, shared_contract: None, live: None, revision: "a".repeat(64), @@ -1002,6 +1059,59 @@ mod tests { assert_eq!(decoded.source, receipt.source); } + #[test] + fn ordinary_replay_propagates_verified_scope_without_adopting_changed_source() { + use std::os::unix::fs::MetadataExt; + let fixture = super::super::tests::Fixture::new(); + fs::write(fixture.0.join("compose.yaml"), "services: {}\n").unwrap(); + let share = super::super::super::ProjectShareIntent::approve(&fixture.0, true).unwrap(); + let metadata = fs::metadata(&fixture.0).unwrap(); + let scope: dependency_cache::ReplayScope = serde_json::from_value(json!({ + "common":fixture.0,"device":metadata.dev(),"inode":metadata.ino(),"original_device":metadata.dev()+7,"origin_sha256":"e".repeat(64), + })).unwrap(); + let mut binding = SourceBinding { + cache_scope: Some(scope.clone()), + shared: Some(share), + shared_contract: None, + live: None, + revision: "a".repeat(64), + archive_sha256: "b".repeat(64), + selection_sha256: "c".repeat(64), + }; + assert!(binding.valid()); + let receipt:Receipt = serde_json::from_value(json!({"version":1,"run":"a".repeat(32),"owner":"b".repeat(32),"namespace":"c".repeat(64),"plan_id":"d".repeat(64),"phase":"stopped-data-retained","readiness":{},"resources":{},"source":binding})).unwrap(); + binding.cache_scope = None; + let mut source = Some(Inputs { + binding, + current_manifest: None, + manifest: ContentRevision { + schema_version: 1, + revision: "a".repeat(64), + selection_sha256: "c".repeat(64), + total_bytes: 0, + entries: vec![], + }, + paths: BTreeMap::new(), + }); + assert!(unchanged(&source, &receipt).is_err()); + inherit_cache_scope(&mut source, &receipt, &fixture.0).unwrap(); + unchanged(&source, &receipt).unwrap(); + verify_cache_scope(source.as_ref(), &fixture.0).unwrap(); + source + .as_mut() + .unwrap() + .binding + .shared + .as_mut() + .unwrap() + .inode += 1; + assert!(unchanged(&source, &receipt).is_err()); + assert!(inherit_cache_scope(&mut None, &receipt, &fixture.0).is_err()); + let foreign = super::super::tests::Fixture::new(); + assert!(verify_cache_scope(source.as_ref(), &foreign.0).is_err()); + assert!(inherit_cache_scope(&mut source, &receipt, &foreign.0).is_err()); + } + #[test] fn shared_source_contract_round_trips_without_rewriting_legacy_bindings() { let fixture = super::super::tests::Fixture::new(); @@ -1033,6 +1143,7 @@ mod tests { shared: Some( super::super::super::ProjectShareIntent::approve(&fixture.0, true).unwrap(), ), + cache_scope: None, shared_contract: Some(contract.clone()), live: None, revision: snapshot.receipt().revision.clone(), diff --git a/packages/runtime-core/src/provider/graph/source_device_rebind.rs b/packages/runtime-core/src/provider/graph/source_device_rebind.rs index c62e4d8d3..f5f5a0872 100644 --- a/packages/runtime-core/src/provider/graph/source_device_rebind.rs +++ b/packages/runtime-core/src/provider/graph/source_device_rebind.rs @@ -1,7 +1,7 @@ //! One explicit source-device translation for an acknowledged stopped graph. //! //! The stopped receipt and its cleanup proofs remain immutable. This witness may -//! project only the share device for source admission into the next generation. +//! project the share device and a proven retained cache scope into the next generation. use super::{ Candidate, CandidateError, Engine, Kind, Receipt, cleanup_enrollment, directory, foreground, host_pin_recovery, inspect_resource, load, restore, source, state, @@ -219,6 +219,19 @@ impl Witness { if *shared != self.current_share { return Err(refused()); } + let scopes = receipt + .resources + .values() + .filter_map(|resource| resource.cache.as_ref().map(|cache| cache.scope.as_str())) + .collect::>(); + let binding = projected.source.as_mut().ok_or_else(refused)?; + binding.cache_scope = super::dependency_cache::ReplayScope::project( + &self.old_share, + &self.current_share, + binding.cache_scope.as_ref(), + &scopes, + &sha256(&encoded(self)?), + )?; Ok(projected) } @@ -237,6 +250,49 @@ impl Witness { } } +/// A later ordinary replay uses the original witness as immutable provenance, +/// never as authority to change the current Owner, guest boot or source device. +pub(super) fn verify_cache_scope_origin( + candidate: &Candidate, + receipt: &Receipt, +) -> Result<(), CandidateError> { + let Some(binding) = &receipt.source else { + return Ok(()); + }; + let Some(scope) = &binding.cache_scope else { + return Ok(()); + }; + let root = directory(candidate, &receipt.run)?; + no_pending(&root)?; + let pin = pin_raw(&root.join(FILE), LIMIT, false)?; + let witness: Witness = serde_json::from_slice(&pin.bytes).map_err(|_| refused())?; + if encoded(&witness)? != pin.bytes + || witness.version != 1 + || witness.run != receipt.run + || witness.owner != receipt.owner + || witness.namespace != receipt.namespace + || witness.plan != receipt.plan_id + || binding.shared.as_ref() != Some(&witness.current_share) + || !scope.matches_origin( + &sha256(&pin.bytes), + &witness.old_share, + &witness.current_share, + ) + { + return Err(refused()); + } + let expected = scope.scope(&witness.current_share.project)?; + if receipt + .resources + .values() + .filter_map(|resource| resource.cache.as_ref()) + .any(|cache| cache.scope != expected) + { + return Err(refused()); + } + pin.reverify() +} + /// Pinned complete witness for the original stopped receipt only. A subsequent /// graph generation cannot use this object as cleanup or source authority. pub(super) struct Selected { @@ -270,7 +326,9 @@ impl Selected { self.pin.reverify()?; validate_current(engine, &self.root, &self.original, &self.witness)?; let current: Receipt = state::read(&self.root.join("state.json"))?; - if encoded(¤t)? != encoded(&self.original)? { + if encoded(¤t)? != encoded(&self.original)? + || encoded(&self.witness.project(¤t)?)? != encoded(&self.projected)? + { return Err(refused()); } Ok(()) @@ -526,6 +584,117 @@ mod tests { file.write_all(bytes).unwrap(); } + #[test] + fn cached_projection_adds_continuity_only_to_the_new_attempt() { + let fixture = super::super::tests::Fixture::new(); + fs::write(fixture.0.join("compose.yaml"), "services: {}\n").unwrap(); + let current_share = ProjectShareIntent::approve(&fixture.0, true).unwrap(); + let mut old_share = current_share.clone(); + old_share.device += 7; + let legacy_scope = sha256( + &serde_json::to_vec(&( + "hack-dependency-cache-scope-v1", + &fixture.0, + old_share.device, + old_share.inode, + )) + .unwrap(), + ); + let receipt: Receipt = serde_json::from_value(json!({ + "version":1,"run":"a".repeat(32),"owner":"b".repeat(32),"namespace":"c".repeat(64), + "plan_id":"d".repeat(64),"phase":"stopped-data-retained","readiness":{}, + "source":{"shared":old_share,"revision":"e".repeat(64),"archive_sha256":"f".repeat(64),"selection_sha256":"0".repeat(64)}, + "resources":{"volume:deps":{"kind":"volume","key":"deps","name":format!("hack-cache-v5-{}","9".repeat(64)),"phase":"present","id":null,"image":null,"cache":{"scope":legacy_scope,"fingerprint":"9".repeat(64),"image":format!("sha256:{}","8".repeat(64))}}} + })).unwrap(); + let original = encoded(&receipt).unwrap(); + let witness = Witness { + version: 1, + run: receipt.run.clone(), + owner: receipt.owner.clone(), + namespace: receipt.namespace.clone(), + plan: receipt.plan_id.clone(), + original_ready_sha256: "1".repeat(64), + stopped_raw_sha256: "2".repeat(64), + absent_intent_raw_sha256: "3".repeat(64), + retirement_raw_sha256: "4".repeat(64), + original_owner_raw_sha256: "5".repeat(64), + current_owner_raw_sha256: "6".repeat(64), + host_boot_micros: 7, + previous_guest_boot: "old".into(), + current_guest_boot: "new".into(), + old_share, + current_share, + retained_volume_projections: BTreeMap::new(), + retained_volumes: BTreeMap::new(), + }; + let projected = witness.project(&receipt).unwrap(); + let scope = projected + .source + .as_ref() + .unwrap() + .cache_scope + .as_ref() + .unwrap(); + assert_eq!(scope.scope(&fixture.0).unwrap(), legacy_scope); + assert_eq!( + projected.source.as_ref().unwrap().shared, + Some(witness.current_share.clone()) + ); + assert!(super::super::same_resource_bindings( + &projected.resources, + &receipt.resources + )); + assert_eq!(encoded(&receipt).unwrap(), original); + let decoded: Receipt = serde_json::from_slice(&encoded(&projected).unwrap()).unwrap(); + assert_eq!(decoded.source, projected.source); + + let home = super::super::tests::Fixture::new(); + let candidate = Candidate::discover(&home.0).unwrap(); + let root = directory(&candidate, &receipt.run).unwrap(); + state::private_directory(&root).unwrap(); + assert!(verify_cache_scope_origin(&candidate, &projected).is_err()); + private_file(&root.join(FILE), &encoded(&witness).unwrap()); + verify_cache_scope_origin(&candidate, &projected).unwrap(); + let mut changed_plan = projected.clone(); + changed_plan.plan_id = "7".repeat(64); + assert!(verify_cache_scope_origin(&candidate, &changed_plan).is_err()); + let mut foreign_origin = witness.clone(); + foreign_origin.owner = "7".repeat(32); + fs::remove_file(root.join(FILE)).unwrap(); + private_file(&root.join(FILE), &encoded(&foreign_origin).unwrap()); + assert!(verify_cache_scope_origin(&candidate, &projected).is_err()); + fs::remove_file(root.join(FILE)).unwrap(); + private_file(&root.join(FILE), &encoded(&witness).unwrap()); + let mut foreign_cache = projected.clone(); + foreign_cache + .resources + .get_mut("volume:deps") + .unwrap() + .cache + .as_mut() + .unwrap() + .scope = "0".repeat(64); + assert!(verify_cache_scope_origin(&candidate, &foreign_cache).is_err()); + private_file(&root.join(FILE).with_extension("pending"), b"partial"); + assert!(verify_cache_scope_origin(&candidate, &projected).is_err()); + assert_eq!( + fs::read(root.join(FILE).with_extension("pending")).unwrap(), + b"partial" + ); + + let mut foreign = receipt.clone(); + foreign + .resources + .get_mut("volume:deps") + .unwrap() + .cache + .as_mut() + .unwrap() + .scope = "0".repeat(64); + assert!(witness.project(&foreign).is_err()); + assert_eq!(encoded(&receipt).unwrap(), original); + } + #[test] fn pinned_witness_refuses_equal_bytes_at_a_replacement_path_or_parent() { let fixture = super::super::tests::Fixture::new(); diff --git a/packages/runtime-core/src/provider/graph/tests.rs b/packages/runtime-core/src/provider/graph/tests.rs index 3e2b4a6f1..aa61c788c 100644 --- a/packages/runtime-core/src/provider/graph/tests.rs +++ b/packages/runtime-core/src/provider/graph/tests.rs @@ -1812,6 +1812,7 @@ fn dependency_cache_graphs_share_only_verified_binding_and_pin_restore_targets() current_manifest: None, binding: SourceBinding { shared: None, + cache_scope: None, shared_contract: None, live: None, revision: manifest.revision.clone(), @@ -2181,6 +2182,7 @@ fn shared_source_preserves_app_writes_but_freezes_cache_installer_inputs() { current_manifest: None, binding: SourceBinding { shared: Some(share), + cache_scope: None, shared_contract: Some( project::live_source::Contract::from_plan(&review.plan, &manifest).unwrap(), ), From ee4bf3a2e67c6144f87f7aae8d6ea5398ba1d881 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 12:25:16 -0400 Subject: [PATCH 21/25] fix(runtime): archive retired dependency rebind before restore --- packages/runtime-core/README.md | 11 + .../graph/absent_publication_cleanup.rs | 3 + .../rebind_history.rs | 267 ++++++++++++++++++ .../src/provider/graph/foreground.rs | 1 + .../provider/graph/foreground/transport.rs | 1 + .../graph/foreground/transport_tests.rs | 24 ++ .../src/provider/graph/restore.rs | 19 ++ .../provider/graph/source_device_rebind.rs | 58 ++++ .../src/provider/graph/startup.rs | 19 ++ .../src/provider/graph/startup/native_test.rs | 2 + .../src/provider/graph/startup/runtime.rs | 19 ++ .../provider/graph/startup/runtime/rebind.rs | 6 +- .../graph/startup/runtime/rebind/archive.rs | 52 +++- .../graph/startup/runtime/rebind/tests.rs | 118 ++++++++ 14 files changed, 598 insertions(+), 2 deletions(-) create mode 100644 packages/runtime-core/src/provider/graph/absent_publication_cleanup/rebind_history.rs diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 0bb4599ae..b9f32600c 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -870,3 +870,14 @@ Missing, changed or pending provenance and replaced Git metadata refuse. A secon device transition requires a separately supported explicit recovery; the runtime does not silently extend this projection. Older candidate binaries may refuse the new optional source metadata, so a downgrade must be qualified separately. + +Completed post-reboot absence recovery also retires the prior boot's completed +dependency-rebind journal. A retained foreground restore can finish this archival +for an earlier candidate: it selects the exact original ready receipt and exact +completed stopped receipt from bounded history, verifies durable retirement, +unchanged provider/boot and retained volumes, and proves old and current compute +absent under the held foreground lock and provider lease. The journal's completed +generation must match the original receipt. Original bytes remain in a digest-bound, +resumable archive; a later current-boot journal is preserved for its own cleanup. +Changed evidence, pending state or live compute refuses archival. This does not +replay old dependencies or relax the fresh graph's endpoint/readiness checks. diff --git a/packages/runtime-core/src/provider/graph/absent_publication_cleanup.rs b/packages/runtime-core/src/provider/graph/absent_publication_cleanup.rs index df5007fa4..135ffb7e7 100644 --- a/packages/runtime-core/src/provider/graph/absent_publication_cleanup.rs +++ b/packages/runtime-core/src/provider/graph/absent_publication_cleanup.rs @@ -22,6 +22,8 @@ use std::{ const INTENT: &str = "absent-publication-cleanup.json"; const RETIREMENT: &str = "absent-publication-retirement.json"; const LIMIT: u64 = 2 * 1024 * 1024; +mod rebind_history; +pub(super) use rebind_history::archive_retired_rebind_under; fn refused() -> CandidateError { CandidateError::new( @@ -1168,6 +1170,7 @@ pub fn recover( dependency_slots::recover_cleaned(candidate, ¤t, None)?; } retire(candidate, &engine, &intent, ¤t, &root, &reservation)?; + archive_retired_rebind_under(candidate, &engine, ¤t, &|| reservation.verify())?; Ok(json!({ "run": run, "phase": "stopped-data-retained", diff --git a/packages/runtime-core/src/provider/graph/absent_publication_cleanup/rebind_history.rs b/packages/runtime-core/src/provider/graph/absent_publication_cleanup/rebind_history.rs new file mode 100644 index 000000000..f5391d8a8 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/absent_publication_cleanup/rebind_history.rs @@ -0,0 +1,267 @@ +//! Metadata retirement for a completed prior-boot rebind. Historical receipts +//! select preserved evidence only; current ownership and live absence still +//! fence every archive write/rename under the caller's foreground lock. +use super::*; + +fn completion(root: &Path, current: &Receipt, intent: &Intent) -> Result { + let complete = intent.complete_sha256.as_deref().ok_or_else(refused)?; + let retirement: Retirement = state::read_bounded(&root.join(RETIREMENT), 65536)?; + if retirement.version != 1 + || retirement.selection_sha256 != intent.selection_sha256 + || retirement.complete_sha256 != complete + || retirement.run != current.run + || retirement.owner != current.owner + || current.run != intent.original.run + || current.owner != intent.original.owner + || current.namespace != intent.original.namespace + || current.plan_id != intent.original.plan_id + || current.phase != "stopped-data-retained" + { + return Err(refused()); + } + let completed = + if digest(&serde_json::to_vec_pretty(current).map_err(|_| refused())?) == complete { + current.clone() + } else { + super::super::restore_history::completed_for_recovery(root, current, complete)? + .ok_or_else(refused)? + }; + if intent.original.phase != "ready-observed" + || completed.phase != "stopped-data-retained" + || !intent.selection.matches_graph(&completed) + || dead_owner_cleanup::immutable(&completed)? + != dead_owner_cleanup::immutable(&intent.original)? + || intent.original.resources.len() != completed.resources.len() + || intent.original.resources.iter().any(|(key, before)| { + completed.resources.get(key).is_none_or(|after| { + before.kind != after.kind + || before.id != after.id + || before.name != after.name + || (after.kind != Kind::Volume && after.phase != "absent") + }) + }) + || completed + .resources + .iter() + .filter(|(_, r)| r.kind == Kind::Volume) + .any(|(key, old)| { + current.resources.get(key).is_none_or(|now| { + now.kind != Kind::Volume || old.name != now.name || old.cache != now.cache + }) + }) + || completed + .resources + .values() + .filter(|r| r.kind == Kind::Volume) + .count() + != current + .resources + .values() + .filter(|r| r.kind == Kind::Volume) + .count() + { + return Err(refused()); + } + Ok(completed) +} + +fn compute_absent(engine: &Engine<'_>, receipt: &Receipt) -> Result<(), CandidateError> { + for resource in receipt + .resources + .values() + .filter(|r| r.kind != Kind::Volume) + { + let mut by_name = resource.clone(); + by_name.id = None; + if resource.phase != "absent" + || inspect_resource(engine, receipt, resource)?.is_some() + || inspect_resource(engine, receipt, &by_name)?.is_some() + { + return Err(refused()); + } + } + Ok(()) +} + +/// Called after completed absence retirement, or before the first effect of a +/// newly selected retained restore. The verifier must bind the held foreground +/// lock descriptor to its pathname, including after interrupted archival. +pub(in crate::provider::graph) fn archive_retired_rebind_under( + candidate: &Candidate, + engine: &Engine<'_>, + current: &Receipt, + verify_lock: &dyn Fn() -> Result<(), CandidateError>, +) -> Result<(), CandidateError> { + let root = directory(candidate, ¤t.run)?; + let Some(intent) = read_intent(&root)? else { + return Ok(()); + }; + let boot = startup::dependency_rebind_boot(&root, current)?; + // A later current-boot journal belongs to its own normal cleanup. The old + // absence intent cannot authorize retiring that generation. + if boot.as_deref() == Some(engine.guest().boot_id()) { + return Ok(()); + } + let generation = super::super::service_exec_generation(&intent.original)?; + let history = root.join(format!("dependency-rebind-history-{generation}")); + if boot.is_none() && absent(&root.join("dependency-rebind.pending"))? && absent(&history)? { + return Ok(()); + } + if boot + .as_deref() + .is_some_and(|boot| boot != intent.selection.previous_guest_boot) + { + return Err(refused()); + } + let completed = completion(&root, current, &intent)?; + let current_bytes = serde_json::to_vec_pretty(current).map_err(|_| refused())?; + let intent_bytes = host_pin_recovery::read_raw(&root.join(INTENT), 4 * 1024 * 1024)?; + let retirement_bytes = host_pin_recovery::read_raw(&root.join(RETIREMENT), 65536)?; + if serde_json::to_vec_pretty(&intent).map_err(|_| refused())? != intent_bytes { + return Err(refused()); + } + let verify = || { + verify_lock()?; + no_pending(&root)?; + for name in [ + "live-owner-cleanup.pending", + "source-device-rebind.pending", + "restore-history.pending", + ] { + require_absent(&root.join(name))?; + } + if host_pin_recovery::read_raw(&root.join("state.json"), LIMIT)? != current_bytes + || host_pin_recovery::read_raw(&root.join(INTENT), 4 * 1024 * 1024)? != intent_bytes + || host_pin_recovery::read_raw(&root.join(RETIREMENT), 65536)? != retirement_bytes + || serde_json::to_vec_pretty(&completion(&root, current, &intent)?) + .map_err(|_| refused())? + != serde_json::to_vec_pretty(&completed).map_err(|_| refused())? + { + return Err(refused()); + } + let selection = &intent.selection; + if selection.candidate != candidate.checkout || selection.version != 1 { + return Err(refused()); + } + let old_bytes = private_input(&selection.original_owner_path, 1024 * 1024)?; + let inspection_bytes = private_input(&selection.filesystem_inspection_path, 65536)?; + if digest(&old_bytes) != selection.original_owner_sha256 + || digest(&inspection_bytes) != selection.filesystem_inspection_sha256 + { + return Err(refused()); + } + let old: Owner = serde_json::from_slice(&old_bytes).map_err(|_| refused())?; + let inspection: FilesystemInspection = + serde_json::from_slice(&inspection_bytes).map_err(|_| refused())?; + inspection.verify(&old_bytes, &old)?; + let (owner, owner_sha) = current_owner(candidate, &old, &inspection, engine)?; + if owner_sha != selection.current_owner_sha256 + || owner.token != current.owner + || owner.guest_boot_id.as_deref() != Some(&selection.current_guest_boot) + || owner.previous_guest_boot_id.as_deref() != Some(&selection.previous_guest_boot) + || inspection.host_boot_micros != selection.host_boot_micros + || inspection.old_device != selection.old_device + || inspection.new_device != selection.new_device + { + return Err(refused()); + } + super::super::source_device_rebind::verify_cache_scope_cleanup_origin( + candidate, + current, + &digest(&intent_bytes), + &digest(&retirement_bytes), + )?; + host_pin_recovery::verify_volume_projections( + engine, + &completed, + &selection.retained_volumes, + )?; + host_pin_recovery::verify_volume_projections(engine, current, &selection.retained_volumes)?; + compute_absent(engine, &completed)?; + compute_absent(engine, current)?; + verify_lock() + }; + startup::archive_retired_dependency_rebind( + &root, + &intent.original, + &completed, + &intent.selection.previous_guest_boot, + &verify, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn historical_completion_requires_exact_stopped_digest_and_retirement() { + let fixture = crate::provider::graph::tests::Fixture::new(); + let root = &fixture.0; + let original: Receipt = serde_json::from_value(json!({ + "version":1,"run":"a".repeat(32),"owner":"b".repeat(32),"namespace":"c".repeat(64), + "plan_id":"d".repeat(64),"phase":"ready-observed","readiness":{}, + "resources":{"container:web":{"kind":"container","key":"web","name":"owned-web","id":"e".repeat(64),"image":null,"phase":"present"}, + "volume:data":{"kind":"volume","key":"data","name":"owned-data","id":null,"image":null,"phase":"present"}} + })).unwrap(); + let mut stopped = original.clone(); + stopped.phase = "stopped-data-retained".into(); + stopped.resources.get_mut("container:web").unwrap().phase = "absent".into(); + let mut current = stopped.clone(); + current.resources.get_mut("container:web").unwrap().id = Some("f".repeat(64)); + let selection: Selection = serde_json::from_value(json!({ + "version":1,"candidate":root,"run":original.run,"owner":original.owner, + "namespace":original.namespace,"plan":original.plan_id, + "original_owner_path":"/private/owner","original_owner_sha256":"1".repeat(64), + "filesystem_inspection_path":"/private/inspection","filesystem_inspection_sha256":"2".repeat(64), + "current_owner_sha256":"3".repeat(64),"graph_sha256":digest(&serde_json::to_vec_pretty(&original).unwrap()), + "host_boot_micros":1,"old_device":2,"new_device":3,"previous_guest_boot":"old","current_guest_boot":"current", + "foreground_root":"/private/foreground","control_root":"/private/control","source_shared":null, + "environment_inventory":{},"scoped_bridge_projection":{},"retained_volumes":{},"dependency_reservation":null,"qualification":"test" + })).unwrap(); + let intent = Intent { version:1,selection_sha256:selection.digest().unwrap(),selection, + original:original.clone(),environment:json!({}),bridges:serde_json::from_value(json!({ + "version":1,"owner":original.owner,"boot":"current","run":original.run,"plan":original.plan_id,"capacity":1,"serial":1,"selected":{} + })).unwrap(),prior_bridges:None,complete_sha256:Some(digest(&serde_json::to_vec_pretty(&stopped).unwrap())) }; + let retirement = Retirement { + version: 1, + selection_sha256: intent.selection_sha256.clone(), + complete_sha256: intent.complete_sha256.clone().unwrap(), + owner: current.owner.clone(), + run: current.run.clone(), + }; + state::write(&root.join(RETIREMENT), &retirement).unwrap(); + assert!(completion(root, ¤t, &intent).is_err()); + crate::provider::graph::restore_history::retain(root, &stopped).unwrap(); + assert_eq!( + serde_json::to_vec(&completion(root, ¤t, &intent).unwrap()).unwrap(), + serde_json::to_vec(&stopped).unwrap() + ); + for change in 0..4 { + let mut changed = current.clone(); + match change { + 0 => changed.owner = "0".repeat(32), + 1 => changed.plan_id = "0".repeat(64), + 2 => changed.phase = "ready-observed".into(), + _ => changed.resources.get_mut("volume:data").unwrap().name = "foreign-data".into(), + } + assert!(completion(root, &changed, &intent).is_err()); + } + let mut changed = retirement.clone(); + changed.selection_sha256 = "0".repeat(64); + state::write(&root.join(RETIREMENT), &changed).unwrap(); + assert!(completion(root, ¤t, &intent).is_err()); + state::write(&root.join(RETIREMENT), &retirement).unwrap(); + let mut changed = intent.clone(); + changed.complete_sha256 = Some("0".repeat(64)); + assert!(completion(root, ¤t, &changed).is_err()); + let mut changed = intent.clone(); + changed + .original + .resources + .get_mut("container:web") + .unwrap() + .id = Some("0".repeat(64)); + assert!(completion(root, ¤t, &changed).is_err()); + } +} diff --git a/packages/runtime-core/src/provider/graph/foreground.rs b/packages/runtime-core/src/provider/graph/foreground.rs index 5e1085959..e602f8906 100644 --- a/packages/runtime-core/src/provider/graph/foreground.rs +++ b/packages/runtime-core/src/provider/graph/foreground.rs @@ -237,6 +237,7 @@ fn serve_input<'a>( deadline, &mut runtime, generation, + &|| publication.verify(), ) } else if let Some((compose, _)) = normalized { super::run_normalized_with_host_dependencies_until( diff --git a/packages/runtime-core/src/provider/graph/foreground/transport.rs b/packages/runtime-core/src/provider/graph/foreground/transport.rs index fadb1ee40..ad9b03945 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport.rs @@ -843,6 +843,7 @@ impl Publication { self.listener.as_raw_fd() } pub fn verify(&self) -> Result<(), CandidateError> { + super::super::host_pin_recovery::exact_lock_path(&self.pin.root, &self._lock)?; self.pin.verify() } pub fn accept(&self) -> Result, CandidateError> { diff --git a/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs b/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs index ba48e5f24..f7b3a344d 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs @@ -1,6 +1,30 @@ use super::*; use serde_json::json; +#[test] +fn publication_refuses_a_replaced_operation_lock_and_preserves_it() { + let fixture = super::super::super::tests::Fixture::new(); + let candidate = Candidate::discover(&fixture.0).unwrap(); + let run = "a".repeat(32); + let publication = Publication::bind(&candidate, &run).unwrap(); + publication.verify().unwrap(); + let directory = root(&candidate, &run).unwrap(); + fs::rename( + directory.join("operation.lock"), + directory.join("original.lock"), + ) + .unwrap(); + let replacement = state::Lock::acquire(&directory).unwrap(); + let replaced = fs::read(directory.join("operation.lock")).unwrap(); + assert!(publication.verify().is_err()); + assert_eq!( + fs::read(directory.join("operation.lock")).unwrap(), + replaced + ); + assert!(directory.join("original.lock").exists()); + drop(replacement); +} + #[test] fn pool_gate_excludes_publication_before_engine_for_every_run() { let fixture = super::super::super::tests::Fixture::new(); diff --git a/packages/runtime-core/src/provider/graph/restore.rs b/packages/runtime-core/src/provider/graph/restore.rs index 32932896e..f60b9dfab 100644 --- a/packages/runtime-core/src/provider/graph/restore.rs +++ b/packages/runtime-core/src/provider/graph/restore.rs @@ -71,6 +71,8 @@ pub(super) struct FreshOwnerRestore<'a> { pub identity: NormalizedInputIdentity, pub generation: &'a str, pub deadline: std::time::Instant, + #[cfg(target_os = "macos")] + pub verify_publication: &'a dyn Fn() -> Result<(), CandidateError>, } #[cfg(target_os = "macos")] pub(super) fn restore_normalized_foreground( @@ -80,6 +82,7 @@ pub(super) fn restore_normalized_foreground( deadline: std::time::Instant, runtime: &mut HostRelayRuntime, generation: &str, + verify_publication: &dyn Fn() -> Result<(), CandidateError>, ) -> Result { startup::Driver::check_cancelled(runtime)?; check_environment_deadline(deadline)?; @@ -109,6 +112,7 @@ pub(super) fn restore_normalized_foreground( identity, generation, deadline, + verify_publication, }), ) } @@ -439,6 +443,15 @@ fn restore_inputs( source::verify_cache_scope(source, options.project.project)?; #[cfg(target_os = "macos")] super::source_device_rebind::verify_cache_scope_origin(candidate, &receipt)?; + #[cfg(target_os = "macos")] + if let Some(fresh) = &fresh { + super::absent_publication_cleanup::archive_retired_rebind_under( + candidate, + &engine, + &receipt, + fresh.verify_publication, + )?; + } // Retain the complete acknowledged old generation before replacing its // boot-bound dependency owner. Historical cleanup is verified in its own context. if fresh.is_some() { @@ -572,6 +585,8 @@ mod tests { identity: next.normalized_input.unwrap(), generation: &generation, deadline: Instant::now() + Duration::from_secs(30), + #[cfg(target_os = "macos")] + verify_publication: &|| Ok(()), }; let reviewed = "8".repeat(64); assert!(verify_fresh(&receipt, &reviewed, &fresh, &generation).is_err()); @@ -614,6 +629,8 @@ mod tests { identity: identity.clone(), generation: &generation, deadline: Instant::now() + Duration::from_secs(30), + #[cfg(target_os = "macos")] + verify_publication: &|| Ok(()), }; verify_fresh(&receipt, &receipt.plan_id, &fresh, &generation).unwrap(); assert!(verify_fresh(&receipt, &"2".repeat(64), &fresh, &generation).is_err()); @@ -648,6 +665,8 @@ mod tests { identity: identity.clone(), generation: &generation, deadline: Instant::now() + Duration::from_secs(30), + #[cfg(target_os = "macos")] + verify_publication: &|| Ok(()), }; let changed = "2".repeat(64); assert!(verify_fresh(&receipt, &changed, &fresh, &generation).is_err()); diff --git a/packages/runtime-core/src/provider/graph/source_device_rebind.rs b/packages/runtime-core/src/provider/graph/source_device_rebind.rs index f5f5a0872..80013a21d 100644 --- a/packages/runtime-core/src/provider/graph/source_device_rebind.rs +++ b/packages/runtime-core/src/provider/graph/source_device_rebind.rs @@ -293,6 +293,39 @@ pub(super) fn verify_cache_scope_origin( pin.reverify() } +/// Historical cache provenance also pins the exact cleanup/retirement bytes +/// used to archive its old journal; another same-schema proof is insufficient. +pub(super) fn verify_cache_scope_cleanup_origin( + candidate: &Candidate, + receipt: &Receipt, + intent_sha256: &str, + retirement_sha256: &str, +) -> Result<(), CandidateError> { + verify_cache_scope_origin(candidate, receipt)?; + let Some(scope) = receipt + .source + .as_ref() + .and_then(|source| source.cache_scope.as_ref()) + else { + return Ok(()); + }; + let root = directory(candidate, &receipt.run)?; + let pin = pin_raw(&root.join(FILE), LIMIT, false)?; + let witness: Witness = serde_json::from_slice(&pin.bytes).map_err(|_| refused())?; + if encoded(&witness)? != pin.bytes + || !scope.matches_origin( + &sha256(&pin.bytes), + &witness.old_share, + &witness.current_share, + ) + || witness.absent_intent_raw_sha256 != intent_sha256 + || witness.retirement_raw_sha256 != retirement_sha256 + { + return Err(refused()); + } + pin.reverify() +} + /// Pinned complete witness for the original stopped receipt only. A subsequent /// graph generation cannot use this object as cleanup or source authority. pub(super) struct Selected { @@ -655,6 +688,31 @@ mod tests { assert!(verify_cache_scope_origin(&candidate, &projected).is_err()); private_file(&root.join(FILE), &encoded(&witness).unwrap()); verify_cache_scope_origin(&candidate, &projected).unwrap(); + verify_cache_scope_cleanup_origin( + &candidate, + &projected, + &witness.absent_intent_raw_sha256, + &witness.retirement_raw_sha256, + ) + .unwrap(); + assert!( + verify_cache_scope_cleanup_origin( + &candidate, + &projected, + &"7".repeat(64), + &witness.retirement_raw_sha256 + ) + .is_err() + ); + assert!( + verify_cache_scope_cleanup_origin( + &candidate, + &projected, + &witness.absent_intent_raw_sha256, + &"7".repeat(64) + ) + .is_err() + ); let mut changed_plan = projected.clone(); changed_plan.plan_id = "7".repeat(64); assert!(verify_cache_scope_origin(&candidate, &changed_plan).is_err()); diff --git a/packages/runtime-core/src/provider/graph/startup.rs b/packages/runtime-core/src/provider/graph/startup.rs index fd046759f..e37507447 100644 --- a/packages/runtime-core/src/provider/graph/startup.rs +++ b/packages/runtime-core/src/provider/graph/startup.rs @@ -401,6 +401,25 @@ pub(super) fn archive_dependency_rebind_after_cleanup( runtime::archive_dependency_rebind_after_cleanup(root, original, cleaned, boot) } +#[cfg(target_os = "macos")] +pub(super) fn dependency_rebind_boot( + root: &Path, + receipt: &Receipt, +) -> Result, CandidateError> { + runtime::dependency_rebind_boot(root, receipt) +} + +#[cfg(target_os = "macos")] +pub(super) fn archive_retired_dependency_rebind( + root: &Path, + original: &Receipt, + cleaned: &Receipt, + boot: &str, + verify: &dyn Fn() -> Result<(), CandidateError>, +) -> Result<(), CandidateError> { + runtime::archive_retired_dependency_rebind(root, original, cleaned, boot, verify) +} + pub(super) fn guest_directory(run: &str, generation: &str) -> String { format!("/storage/hack-graph-startup/{run}/{generation}") } diff --git a/packages/runtime-core/src/provider/graph/startup/native_test.rs b/packages/runtime-core/src/provider/graph/startup/native_test.rs index 93e430abc..53b5782c1 100644 --- a/packages/runtime-core/src/provider/graph/startup/native_test.rs +++ b/packages/runtime-core/src/provider/graph/startup/native_test.rs @@ -795,6 +795,8 @@ fn qualify_active_rebind(refreshable: bool, cancel_after_fence: bool) { Instant::now() + Duration::from_secs(60), &mut fresh, &restored_generation, + // This owned driver fixture has no historical absence intent. + &|| Ok(()), ); let assertions = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { let restored = restored.as_ref().unwrap(); diff --git a/packages/runtime-core/src/provider/graph/startup/runtime.rs b/packages/runtime-core/src/provider/graph/startup/runtime.rs index 5025035d8..f6f3384a6 100644 --- a/packages/runtime-core/src/provider/graph/startup/runtime.rs +++ b/packages/runtime-core/src/provider/graph/startup/runtime.rs @@ -28,6 +28,25 @@ pub(super) fn archive_dependency_rebind_after_cleanup( ) -> Result<(), CandidateError> { rebind::archive_after_cleanup(root, original, cleaned, boot) } + +#[cfg(target_os = "macos")] +pub(super) fn dependency_rebind_boot( + root: &Path, + receipt: &Receipt, +) -> Result, CandidateError> { + rebind::boot(root, receipt) +} + +#[cfg(target_os = "macos")] +pub(super) fn archive_retired_dependency_rebind( + root: &Path, + original: &Receipt, + cleaned: &Receipt, + boot: &str, + verify: &dyn Fn() -> Result<(), CandidateError>, +) -> Result<(), CandidateError> { + rebind::archive::retired_completed(root, original, cleaned, boot, verify) +} use sha2::{Digest, Sha256}; use std::{ io::Read, diff --git a/packages/runtime-core/src/provider/graph/startup/runtime/rebind.rs b/packages/runtime-core/src/provider/graph/startup/runtime/rebind.rs index fa0da765c..a0b347b36 100644 --- a/packages/runtime-core/src/provider/graph/startup/runtime/rebind.rs +++ b/packages/runtime-core/src/provider/graph/startup/runtime/rebind.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; use std::{collections::BTreeSet, fs, path::PathBuf}; const JOURNAL: &str = "dependency-rebind.json"; -mod archive; +pub(super) mod archive; #[cfg(test)] mod nested_tests; pub(super) use archive::after_cleanup as archive_after_cleanup; @@ -331,6 +331,10 @@ fn existing(root: &Path, receipt: &Receipt) -> Result, Can } } } +#[cfg(target_os = "macos")] +pub(super) fn boot(root: &Path, receipt: &Receipt) -> Result, CandidateError> { + Ok(existing(root, receipt)?.map(|journal| journal.boot)) +} pub(super) fn require_complete(root: &Path, receipt: &Receipt) -> Result<(), CandidateError> { if pending_journal(root)? { return Err(stage_refused("graph_dependency_rebind_incomplete")); diff --git a/packages/runtime-core/src/provider/graph/startup/runtime/rebind/archive.rs b/packages/runtime-core/src/provider/graph/startup/runtime/rebind/archive.rs index 374cc0aaf..8bd987483 100644 --- a/packages/runtime-core/src/provider/graph/startup/runtime/rebind/archive.rs +++ b/packages/runtime-core/src/provider/graph/startup/runtime/rebind/archive.rs @@ -77,6 +77,49 @@ pub(in crate::provider::graph::startup::runtime) fn after_cleanup( cleaned: &Receipt, boot: &str, ) -> Result<(), CandidateError> { + after_cleanup_fenced(root, original, cleaned, boot, &|| Ok(())) +} + +/// A completed prior-boot journal must match the selected original ready +/// generation before first archival admission. A durable proof alone permits +/// exact rename recovery after the active journal has already moved. +#[cfg(any(target_os = "macos", test))] +pub(in crate::provider::graph::startup::runtime) fn retired_completed( + root: &Path, + original: &Receipt, + cleaned: &Receipt, + boot: &str, + verify: &dyn Fn() -> Result<(), CandidateError>, +) -> Result<(), CandidateError> { + verify()?; + let generation = service_exec_generation(original)?; + let proof = root.join(format!("dependency-rebind-history-{generation}/proof.json")); + let journal_root = if bytes(&root.join(JOURNAL))?.is_some() { + root.to_owned() + } else { + proof.parent().ok_or_else(rejected)?.to_owned() + }; + let journal = existing(&journal_root, original)?.ok_or_else(rejected)?; + if journal.phase != "completed" + || journal.boot != boot + || journal.completed_generation.as_deref() != Some(generation.as_str()) + { + return Err(rejected()); + } + if bytes(&proof)?.is_none() { + require_complete(root, original)?; + } + after_cleanup_fenced(root, original, cleaned, boot, verify) +} + +fn after_cleanup_fenced( + root: &Path, + original: &Receipt, + cleaned: &Receipt, + boot: &str, + verify: &dyn Fn() -> Result<(), CandidateError>, +) -> Result<(), CandidateError> { + verify()?; state::check_private_directory(root)?; if boot.is_empty() || original.run != cleaned.run @@ -118,6 +161,7 @@ pub(in crate::provider::graph::startup::runtime) fn after_cleanup( state::private_directory(&history)?; // A partial proof write is preserved before the same exact owned cleanup // reconstructs it. It is never interpreted as a capability or state receipt. + verify()?; crate::provider::graph::journal::retain_file( &history, "proof.pending", @@ -160,6 +204,7 @@ pub(in crate::provider::graph::startup::runtime) fn after_cleanup( artifacts: active, complete: false, }; + verify()?; state::write(&proof_path, &proof)?; proof }; @@ -179,6 +224,10 @@ pub(in crate::provider::graph::startup::runtime) fn after_cleanup( if digest(&value) != proof.artifacts[name] { return Err(rejected()); } + verify()?; + if bytes(&root.join(name))?.as_deref() != Some(value.as_slice()) { + return Err(rejected()); + } fs::rename(root.join(name), history.join(name)).map_err(state::io)?; sync(&history)?; sync(root)?; @@ -186,7 +235,8 @@ pub(in crate::provider::graph::startup::runtime) fn after_cleanup( } if !proof.complete { proof.complete = true; + verify()?; state::write(&proof_path, &proof)?; } - Ok(()) + verify() } diff --git a/packages/runtime-core/src/provider/graph/startup/runtime/rebind/tests.rs b/packages/runtime-core/src/provider/graph/startup/runtime/rebind/tests.rs index e50efa991..0900cce89 100644 --- a/packages/runtime-core/src/provider/graph/startup/runtime/rebind/tests.rs +++ b/packages/runtime-core/src/provider/graph/startup/runtime/rebind/tests.rs @@ -672,6 +672,124 @@ fn terminal_refresh_proofs_do_not_pin_old_helpers_across_owned_restore() { fs::remove_dir_all(root).unwrap(); } +#[test] +fn retired_completed_rebind_requires_exact_generation_and_resumes_from_proof() { + let fixture = crate::provider::graph::tests::Fixture::new(); + let root = &fixture.0; + let mut original = receipt(); + original + .readiness + .insert("init".into(), Condition::Completed); + original.relay_startup = Some(startup(&"6".repeat(64), &["web", "init"])); + let init = original + .relay_startup + .as_mut() + .unwrap() + .services + .get_mut("init") + .unwrap(); + init.phase = Phase::Completed; + init.bindings.get_mut("content").unwrap().process = None; + let process = original.relay_startup.as_ref().unwrap().services["web"].bindings["content"] + .process + .unwrap(); + original.resources.insert( + "container:web".into(), + serde_json::from_value(json!({ + "kind":"container","key":"web","name":"owned-web","id":"7".repeat(64), + "image":"sha256:".to_string()+&"8".repeat(64),"phase":"present" + })) + .unwrap(), + ); + let mut cleaned = original.clone(); + cleaned.phase = "stopped-data-retained".into(); + cleaned.resources.get_mut("container:web").unwrap().phase = "absent".into(); + let generation = service_exec_generation(&original).unwrap(); + let mut journal = RebindJournal { + version: 1, + operation: "4".repeat(32), + run: original.run.clone(), + owner: original.owner.clone(), + boot: "old-boot".into(), + expected_generation: "5".repeat(64), + phase: "completed".into(), + slots: BTreeMap::from([( + 0, + JournalSlot { + before: "5".repeat(64), + after: "6".repeat(64), + bindings: vec![ + ("web".into(), "content".into()), + ("init".into(), "content".into()), + ], + terminal_only: false, + }, + )]), + processes: BTreeMap::from([("web".into(), BTreeMap::from([("content".into(), process)]))]), + completed_services: BTreeSet::from(["init".into()]), + completed_generation: Some(generation.clone()), + }; + state::write(&root.join(JOURNAL), &journal).unwrap(); + let original_bytes = fs::read(root.join(JOURNAL)).unwrap(); + for wrong_boot in ["current-boot", ""] { + assert!( + archive::retired_completed(root, &original, &cleaned, wrong_boot, &|| Ok(())).is_err() + ); + assert_eq!(fs::read(root.join(JOURNAL)).unwrap(), original_bytes); + } + journal.completed_generation = Some("9".repeat(64)); + state::write(&root.join(JOURNAL), &journal).unwrap(); + assert!(archive::retired_completed(root, &original, &cleaned, "old-boot", &|| Ok(())).is_err()); + journal.completed_generation = Some(generation.clone()); + state::write(&root.join(JOURNAL), &journal).unwrap(); + let history = root.join(format!("dependency-rebind-history-{generation}")); + // Lost foreground ownership before the first write leaves evidence intact. + assert!( + archive::retired_completed(root, &original, &cleaned, "old-boot", &|| Err(rejected())) + .is_err() + ); + assert!(!history.exists()); + let verify = || { + if history.join("proof.json").exists() && root.join(JOURNAL).exists() { + Err(rejected()) + } else { + Ok(()) + } + }; + assert!(archive::retired_completed(root, &original, &cleaned, "old-boot", &verify).is_err()); + assert_eq!(fs::read(root.join(JOURNAL)).unwrap(), original_bytes); + assert_eq!( + state::read::(&history.join("proof.json")).unwrap()["complete"], + false + ); + // Simulate the owned archive's first successful rename followed by a crash. + fs::rename(root.join(JOURNAL), history.join(JOURNAL)).unwrap(); + assert!(require_complete(root, &original).is_err()); + crate::provider::graph::startup::archive_retired_dependency_rebind( + root, + &original, + &cleaned, + "old-boot", + &|| Ok(()), + ) + .unwrap(); + archive::retired_completed(root, &original, &cleaned, "old-boot", &|| Ok(())).unwrap(); + assert_eq!(fs::read(history.join(JOURNAL)).unwrap(), original_bytes); + assert_eq!( + state::read::(&history.join("proof.json")).unwrap()["complete"], + true + ); + require_complete(root, &cleaned).unwrap(); + let mut changed = cleaned.clone(); + changed.resources.get_mut("container:web").unwrap().id = Some("a".repeat(64)); + assert!(archive::retired_completed(root, &original, &changed, "old-boot", &|| Ok(())).is_err()); + let mut proof: Value = state::read(&history.join("proof.json")).unwrap(); + proof["boot"] = json!("substituted-boot"); + state::write(&history.join("proof.json"), &proof).unwrap(); + assert!(archive::retired_completed(root, &original, &cleaned, "old-boot", &|| Ok(())).is_err()); + assert_eq!(fs::read(history.join(JOURNAL)).unwrap(), original_bytes); +} + #[test] fn completed_journal_requires_explicit_terminal_state_and_never_fabricates_helpers() { let root = std::env::temp_dir().canonicalize().unwrap().join(format!( From 0b80466aa14b955540ea1f13a64d804d2ff7717a Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 13:08:28 -0400 Subject: [PATCH 22/25] fix: retire acknowledged stopped graph publishers explicitly --- packages/runtime-core/README.md | 19 ++ packages/runtime-core/src/graph_cli.rs | 43 +++ .../src/graph_cli/restore_tests.rs | 38 +++ packages/runtime-core/src/main.rs | 1 + .../provider/graph/acknowledged_publisher.rs | 300 ++++++++++++++++++ .../provider/graph/foreground/transport.rs | 27 ++ .../graph/foreground/transport_tests.rs | 108 +++++++ .../runtime-core/src/provider/graph/mod.rs | 6 + 8 files changed, 542 insertions(+) create mode 100644 packages/runtime-core/src/provider/graph/acknowledged_publisher.rs diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index b9f32600c..07b75f8c8 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -840,6 +840,25 @@ precedence over older cleanup records. Pending operations, unresolved initialize effects and mismatched receipt or retirement identities still refuse recovery; historical records cannot authorize a later generation. +Ordinary cleanup can be acknowledged before a later dependency-journal archival +step fails. If this leaves a dead foreground publication for the current stopped +graph, use the explicit private `graph retire-acknowledged-publisher --run-id RUN +--expect-owner OWNER --expect-receipt RECEIPT_SHA --expect-publisher PUBLISHER_SHA +--json` recovery. The SHA selectors identify the exact stopped receipt and +foreground owner-record bytes. Admission requires the current guest boot's +confirmed cleanup acknowledgement, no pending operation, independent absence of +compute by both immutable ID and reserved name, and all named volumes present. +Environment, startup, probe and bridge cleanup inventories are rechecked while +holding the publisher and provider locks, including at each publication effect. +The existing retirement journal preserves both original publication files and +resumes an interrupted socket-first rename. Active owners, replacement listeners, +stale selectors and incomplete acknowledgements refuse without falling back to +historical recovery. This operation changes no graph receipt, VM, dependency +journal or retained data; it does not authorize a restore by itself. +An old completed or cleanup-completed dependency journal is accepted as inert +evidence only; malformed, incomplete or pending rebind state refuses at each +effect boundary. Safe archival and the next restore remain separate checks. + `restore-selection` then binds the witness's raw hash to the selected generation. The source checks use only an in-memory device projection; original retention and restore history consume the unchanged stopped receipt. The new receipt receives diff --git a/packages/runtime-core/src/graph_cli.rs b/packages/runtime-core/src/graph_cli.rs index 9bf84625d..30e92adfa 100644 --- a/packages/runtime-core/src/graph_cli.rs +++ b/packages/runtime-core/src/graph_cli.rs @@ -25,6 +25,49 @@ pub fn command(candidate: &Candidate, args: &[&str]) -> Result (run, owner, receipt, publisher), + _ => return Err(invalid()), + }; + #[cfg(target_os = "macos")] + { + return graph::retire_acknowledged_publisher( + candidate, + graph::AcknowledgedPublisherSelection { + run, + owner, + receipt_sha256: receipt, + publisher_sha256: publisher, + }, + ); + } + #[cfg(not(target_os = "macos"))] + { + let _ = (run, owner, receipt, publisher); + return Err(invalid()); + } + } if *action == "inspect-host-pin-recovery" { let run = match *args { ["--run-id", run] | ["--run-id", run, "--json"] => run, diff --git a/packages/runtime-core/src/graph_cli/restore_tests.rs b/packages/runtime-core/src/graph_cli/restore_tests.rs index fc407a4bb..456fa6cce 100644 --- a/packages/runtime-core/src/graph_cli/restore_tests.rs +++ b/packages/runtime-core/src/graph_cli/restore_tests.rs @@ -5,6 +5,44 @@ const RUN: &str = "11111111111111111111111111111111"; const PLAN: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; const GENERATION: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +#[test] +fn acknowledged_retirement_requires_every_selector_and_forbids_other_effects() { + let fixture = Fixture::new(); + let candidate = Candidate::discover(&fixture.0).unwrap(); + let args = vec![ + "retire-acknowledged-publisher", + "--run-id", + RUN, + "--expect-owner", + RUN, + "--expect-receipt", + PLAN, + "--expect-publisher", + GENERATION, + ]; + for offset in [1, 3, 5, 7] { + let mut missing = args.clone(); + missing.drain(offset..offset + 2); + assert_eq!( + command(&candidate, &missing).unwrap_err().code, + "graph_arguments" + ); + } + for effect in [ + "--remove-data", + "--accept-legacy-device-rebind", + "--environment-stdin", + ] { + let mut expanded = args.clone(); + expanded.push(effect); + assert_eq!( + command(&candidate, &expanded).unwrap_err().code, + "graph_arguments" + ); + } + assert!(!candidate.state_root.exists()); +} + #[test] fn same_boot_recovery_requires_explicit_receipt_and_forbids_data_removal() { let fixture = Fixture::new(); diff --git a/packages/runtime-core/src/main.rs b/packages/runtime-core/src/main.rs index a7434f749..3066a79b2 100644 --- a/packages/runtime-core/src/main.rs +++ b/packages/runtime-core/src/main.rs @@ -27,6 +27,7 @@ Usage: hack-local graph recover-cleanup --run-id <32-hex> --expect-receipt [--json] hack-local graph recover-live-owner --run-id <32-hex> --expect-receipt [--json] hack-local graph retire-recovered-publisher --run-id <32-hex> --expect-owner <32-hex> [--json] + hack-local graph retire-acknowledged-publisher --run-id <32-hex> --expect-owner <32-hex> --expect-receipt <64-hex> --expect-publisher <64-hex> [--json] hack-local graph inspect-host-pin-recovery --run-id <32-hex> [--json] hack-local graph recover-host-pins --run-id <32-hex> --expect-selection <64-hex> --accept-legacy-device-rebind [--json] hack-local graph inspect-absent-publication-cleanup --run-id <32-hex> --original-owner-file --host-inspection-file [--json] diff --git a/packages/runtime-core/src/provider/graph/acknowledged_publisher.rs b/packages/runtime-core/src/provider/graph/acknowledged_publisher.rs new file mode 100644 index 000000000..e47f77e5d --- /dev/null +++ b/packages/runtime-core/src/provider/graph/acknowledged_publisher.rs @@ -0,0 +1,300 @@ +//! Publication-only recovery after ordinary cleanup was acknowledged, but a +//! later archival step failed. Historical recovery receipts grant no authority. +use super::*; +use sha2::{Digest, Sha256}; +use std::path::Path; + +const LIMIT: u64 = 2 * 1024 * 1024; + +pub struct AcknowledgedPublisherSelection<'a> { + pub run: &'a str, + pub owner: &'a str, + pub receipt_sha256: &'a str, + pub publisher_sha256: &'a str, +} + +fn refused() -> CandidateError { + error( + "graph_acknowledged_publisher", + "Publisher retirement requires the exact stopped receipt, current-boot cleanup acknowledgement, absent compute and unchanged retained data; evidence preserved.", + ) +} + +fn no_pending(root: &Path) -> Result<(), CandidateError> { + for name in [ + "state.pending", + "restore-history.pending", + "one-off.json", + "one-off.pending", + "one-off-normalization.json", + "dependency-rebind.pending", + "source-device-rebind.pending", + "dead-owner-cleanup.pending", + "live-owner-cleanup.pending", + "absent-publication-cleanup.pending", + "absent-publication-retirement.pending", + "relay-cleanup-bridges.pending", + "retired-data-removal.json", + "retired-data-removal.pending", + ] { + match fs::symlink_metadata(root.join(name)) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + _ => return Err(refused()), + } + } + Ok(()) +} + +fn eligible( + receipt: &Receipt, + selected: &AcknowledgedPublisherSelection<'_>, + boot: &str, +) -> Result<(), CandidateError> { + let marker = receipt.relay_cleanup.as_ref().ok_or_else(refused)?; + let startup = receipt.relay_startup.as_ref().ok_or_else(refused)?; + let context = host_relay::context(selected.owner, boot)?; + if receipt.run != selected.run + || receipt.owner != selected.owner + || receipt.phase != "stopped-data-retained" + || receipt.normalized_input.is_none() + || !marker.valid() + || marker.phase != cleanup_enrollment::Phase::Confirmed + || marker.runtime != context.runtime + || marker.boot != context.boot + || marker.control_root != startup.control_root + || receipt + .resources + .values() + .any(|resource| match resource.kind { + Kind::Volume => resource.phase != "created", + _ => resource.phase != "absent", + }) + { + return Err(refused()); + } + initializer_cache::require_resolved(receipt) +} + +fn exact_receipt(root: &Path, expected: &str) -> Result<(), CandidateError> { + no_pending(root)?; + let bytes = host_pin_recovery::read_raw(&root.join("state.json"), LIMIT)?; + if format!("{:x}", Sha256::digest(bytes)) != expected { + return Err(refused()); + } + Ok(()) +} + +fn acknowledged_effect(receipt: &Receipt, effect: [u8; 32]) -> Result<(), CandidateError> { + if receipt + .relay_cleanup + .as_ref() + .is_none_or(|marker| marker.effect != effect) + { + return Err(refused()); + } + Ok(()) +} + +/// Retire only a selected dead publication after independent current-boot ACK +/// and guest absence checks. The existing immutable publisher journal preserves +/// both original files and resumes either rename interruption. No graph receipt, +/// VM resource, dependency journal or persistent volume is changed here. +pub fn retire( + candidate: &Candidate, + selected: AcknowledgedPublisherSelection<'_>, +) -> Result { + if !hex(selected.run, 32) + || !hex(selected.owner, 32) + || !hex(selected.receipt_sha256, 64) + || !hex(selected.publisher_sha256, 64) + { + return Err(refused()); + } + // Match the existing explicit recovery lock order; both locks stay held + // through observation, journal publication and each retirement rename. + let engine = Engine::connect_cleanup_wait(candidate)?; + let publication = foreground::transport::root(candidate, selected.run)?; + let lock = state::Lock::acquire_existing(&publication)?; + host_pin_recovery::exact_lock_path(&publication, &lock)?; + let (receipt, root) = load(candidate, &engine, selected.run)?; + eligible(&receipt, &selected, engine.guest().boot_id())?; + exact_receipt(&root, selected.receipt_sha256)?; + host_relay::require_acknowledged_enrollment(&receipt)?; + let environment = environment::cleanup_inventory(candidate, &engine, &receipt, &root)?; + let bridges = bridges::cleanup::read(&engine, &receipt, &root)?; + let verify = || { + host_pin_recovery::exact_lock_path(&publication, &lock)?; + exact_receipt(&root, selected.receipt_sha256)?; + startup::require_dependency_rebind_complete(&root, &receipt)?; + acknowledged_effect( + &receipt, + host_relay::cleanup_effect( + &receipt, + engine.guest().boot_id(), + false, + &(&environment, &bridges), + )?, + )?; + host_relay::require_acknowledged_enrollment(&receipt)?; + host_relay::inspect_cleanup(candidate, &engine, &receipt, false, &environment, &bridges)?; + engine.guest().verify() + }; + verify()?; + // This helper checks the selected publisher bytes, dead process, refused + // listener and exact lock/path identities before writing or moving files. + // Its durable journal permits retry after either original name has moved. + foreground::transport::retire_recovered_publisher_locked_fenced( + candidate, + selected.run, + selected.publisher_sha256, + selected.receipt_sha256, + None, + &lock, + &verify, + )?; + verify()?; + Ok( + json!({"run":selected.run,"publisher_retired":true,"data_retained":true,"same_boot":true,"acknowledged_cleanup":true}), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn receipt() -> Receipt { + let mut value: Receipt = serde_json::from_value(json!({"version":1,"run":"a".repeat(32),"owner":"b".repeat(32),"namespace":"c".repeat(64),"plan_id":"d".repeat(64),"phase":"stopped-data-retained","normalized_input":{"namespace":"c".repeat(64),"original_compose_sha256":"d".repeat(64),"normalized_compose_sha256":"e".repeat(64)},"readiness":{},"resources":{"container:web":{"kind":"container","key":"web","name":"owned-web","id":"f".repeat(64),"image":"sha256:".to_owned()+&"f".repeat(64),"phase":"absent"},"volume:data":{"kind":"volume","key":"data","name":"owned-data","id":null,"image":null,"phase":"created"}},"relay_startup":{"control_only":true,"guest_root":null,"control_root":"/private/owned","artifact":"e".repeat(64),"services":{}},"relay_cleanup":{"version":1,"runtime":vec![1;16],"boot":vec![2;16],"operation":vec![3;16],"effect":vec![4;32],"control_root":"/private/owned","phase":"confirmed"}})).unwrap(); + let context = host_relay::context(&value.owner, "fixture-boot").unwrap(); + let marker = value.relay_cleanup.as_mut().unwrap(); + marker.runtime = context.runtime; + marker.boot = context.boot; + value + } + + #[test] + fn acknowledgement_eligibility_refuses_other_generations_and_partial_cleanup() { + let value = receipt(); + let selected = AcknowledgedPublisherSelection { + run: &value.run, + owner: &value.owner, + receipt_sha256: &"1".repeat(64), + publisher_sha256: &"2".repeat(64), + }; + eligible(&value, &selected, "fixture-boot").unwrap(); + assert!(eligible(&value, &selected, "other-boot").is_err()); + for field in [ + "run", + "owner", + "phase", + "normalized", + "marker", + "ack", + "root", + "compute", + "volume", + ] { + let mut changed = value.clone(); + match field { + "run" => changed.run = "9".repeat(32), + "owner" => changed.owner = "9".repeat(32), + "phase" => changed.phase = "failed-retained".into(), + "normalized" => changed.normalized_input = None, + "marker" => changed.relay_cleanup = None, + "ack" => { + changed.relay_cleanup.as_mut().unwrap().phase = + cleanup_enrollment::Phase::Pending + } + "root" => { + changed.relay_startup.as_mut().unwrap().control_root = "/private/other".into() + } + "compute" => { + changed.resources.get_mut("container:web").unwrap().phase = "started".into() + } + "volume" => { + changed.resources.get_mut("volume:data").unwrap().phase = "absent".into() + } + _ => unreachable!(), + } + assert!( + eligible(&changed, &selected, "fixture-boot").is_err(), + "{field}" + ); + } + } + + #[test] + fn pending_operations_and_changed_receipt_bytes_are_preserved() { + let fixture = super::super::tests::Fixture::new(); + let value = receipt(); + state::write(&fixture.0.join("state.json"), &value).unwrap(); + let before = fs::read(fixture.0.join("state.json")).unwrap(); + let selected = format!("{:x}", Sha256::digest(&before)); + exact_receipt(&fixture.0, &selected).unwrap(); + assert!(exact_receipt(&fixture.0, &"0".repeat(64)).is_err()); + for name in [ + "state.pending", + "restore-history.pending", + "dependency-rebind.pending", + "source-device-rebind.pending", + "one-off.json", + "retired-data-removal.json", + "relay-cleanup-bridges.pending", + ] { + fs::write(fixture.0.join(name), b"preserve").unwrap(); + assert!(exact_receipt(&fixture.0, &selected).is_err(), "{name}"); + assert_eq!(fs::read(fixture.0.join(name)).unwrap(), b"preserve"); + assert_eq!(fs::read(fixture.0.join("state.json")).unwrap(), before); + fs::remove_file(fixture.0.join(name)).unwrap(); + } + fs::write(fixture.0.join("state.json"), b"changed").unwrap(); + assert!(exact_receipt(&fixture.0, &selected).is_err()); + assert_eq!(fs::read(fixture.0.join("state.json")).unwrap(), b"changed"); + } + + #[test] + fn completed_old_boot_journal_is_inert_but_uncertain_rebind_refuses() { + let fixture = super::super::tests::Fixture::new(); + let value = receipt(); + let mut journal = json!({"version":1,"operation":"1".repeat(32),"run":value.run,"owner":value.owner,"boot":"prior-boot","expected_generation":"2".repeat(64),"phase":"completed","slots":{},"processes":{},"completed_services":["deps"],"completed_generation":"3".repeat(64)}); + let path = fixture.0.join("dependency-rebind.json"); + for phase in ["completed", "cleaned"] { + journal["phase"] = json!(phase); + state::write(&path, &journal).unwrap(); + let before = fs::read(&path).unwrap(); + startup::require_dependency_rebind_complete(&fixture.0, &value).unwrap(); + assert_eq!(fs::read(&path).unwrap(), before); + fs::write(fixture.0.join("dependency-rebind.pending"), b"partial").unwrap(); + assert!(startup::require_dependency_rebind_complete(&fixture.0, &value).is_err()); + assert_eq!(fs::read(&path).unwrap(), before); + fs::remove_file(fixture.0.join("dependency-rebind.pending")).unwrap(); + } + journal["phase"] = json!("provisioning"); + state::write(&path, &journal).unwrap(); + assert!(startup::require_dependency_rebind_complete(&fixture.0, &value).is_err()); + fs::write(&path, b"malformed").unwrap(); + assert!(startup::require_dependency_rebind_complete(&fixture.0, &value).is_err()); + assert_eq!(fs::read(&path).unwrap(), b"malformed"); + } + + #[test] + fn acknowledgement_is_bound_to_the_selected_cleanup_effect() { + let mut value = receipt(); + let calculate = |receipt: &Receipt| { + host_relay::cleanup_effect(receipt, "fixture-boot", false, &json!(["inventory"])) + .unwrap() + }; + let effect = calculate(&value); + value.relay_cleanup.as_mut().unwrap().effect = effect; + acknowledged_effect(&value, calculate(&value)).unwrap(); + let mut changed = value.clone(); + changed.relay_cleanup.as_mut().unwrap().effect[0] ^= 1; + assert!(acknowledged_effect(&changed, calculate(&changed)).is_err()); + changed = value.clone(); + changed.resources.get_mut("container:web").unwrap().id = Some("9".repeat(64)); + assert!(acknowledged_effect(&changed, calculate(&changed)).is_err()); + let wrong_inventory = + host_relay::cleanup_effect(&value, "fixture-boot", false, &json!(["other-inventory"])) + .unwrap(); + assert!(acknowledged_effect(&value, wrong_inventory).is_err()); + } +} diff --git a/packages/runtime-core/src/provider/graph/foreground/transport.rs b/packages/runtime-core/src/provider/graph/foreground/transport.rs index ad9b03945..42b3205ac 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport.rs @@ -361,6 +361,28 @@ pub(in crate::provider::graph) fn retire_recovered_publisher_locked( device_rebind: Option, lock: &state::Lock, ) -> Result<(), CandidateError> { + retire_recovered_publisher_locked_fenced( + candidate, + run, + expected_owner, + expected_receipt, + device_rebind, + lock, + &|| Ok(()), + ) +} + +/// The caller's current cleanup proof remains valid at each publication effect. +pub(in crate::provider::graph) fn retire_recovered_publisher_locked_fenced( + candidate: &Candidate, + run: &str, + expected_owner: &str, + expected_receipt: &str, + device_rebind: Option, + lock: &state::Lock, + verify_cleanup: &dyn Fn() -> Result<(), CandidateError>, +) -> Result<(), CandidateError> { + verify_cleanup()?; let root = root(candidate, run)?; let path = retirement_path(&root, expected_owner); let pending = path.with_extension("pending"); @@ -368,6 +390,7 @@ pub(in crate::provider::graph) fn retire_recovered_publisher_locked( .file_name() .and_then(|value| value.to_str()) .ok_or_else(retirement_refused)?; + verify_cleanup()?; super::super::journal::retain_file(&root, name, "publisher-retirement-interrupted", 8192)?; let intent: Retirement = if metadata(&path)?.is_some() { state::read(&path).map_err(|_| retirement_refused())? @@ -391,6 +414,7 @@ pub(in crate::provider::graph) fn retire_recovered_publisher_locked( record: pin.record_id, owner: pin.record, }; + verify_cleanup()?; state::write(&path, &selected).map_err(|_| retirement_refused())?; selected }; @@ -404,6 +428,7 @@ pub(in crate::provider::graph) fn retire_recovered_publisher_locked( device_rebind, )?; if socket_original { + verify_cleanup()?; fs::rename( root.join("control.sock"), retired_path(&root, expected_owner, true), @@ -414,6 +439,7 @@ pub(in crate::provider::graph) fn retire_recovered_publisher_locked( .map_err(|_| retirement_refused())?; } if record_original { + verify_cleanup()?; let _ = verify_retirement( candidate, run, @@ -444,6 +470,7 @@ pub(in crate::provider::graph) fn retire_recovered_publisher_locked( if socket_original || record_original { return Err(retirement_refused()); } + verify_cleanup()?; Ok(()) } diff --git a/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs b/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs index f7b3a344d..8b5dac3bc 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs @@ -131,6 +131,114 @@ fn recovered_publisher_retirement_is_exact_and_idempotent() { fs::remove_dir_all(root).unwrap(); } +#[cfg(target_os = "macos")] +#[test] +fn cleanup_fence_refuses_effects_and_resumes_each_retirement_rename() { + for fail_at in [1, 3, 4, 5, 6] { + let (_fixture, candidate, run, owner, directory) = abandoned_publisher(); + let receipt = "f".repeat(64); + let before = fs::read(directory.join("owner.json")).unwrap(); + let sibling_run = "b".repeat(32); + let mut sibling = Publication::bind(&candidate, &sibling_run).unwrap(); + let sibling_root = root(&candidate, &sibling_run).unwrap(); + let sibling_before = fs::read(sibling_root.join("owner.json")).unwrap(); + let lock = state::Lock::acquire_existing(&directory).unwrap(); + let calls = std::cell::Cell::new(0); + let fence = || { + calls.set(calls.get() + 1); + if calls.get() == fail_at { + Err(retirement_refused()) + } else { + Ok(()) + } + }; + assert!( + retire_recovered_publisher_locked_fenced( + &candidate, &run, &owner, &receipt, None, &lock, &fence, + ) + .is_err() + ); + assert_eq!(calls.get(), fail_at); + let owner_path = if fail_at == 6 { + retired_path(&directory, &owner, false) + } else { + directory.join("owner.json") + }; + assert_eq!(fs::read(&owner_path).unwrap(), before); + let journal = retirement_path(&directory, &owner); + assert_eq!(journal.exists(), fail_at >= 4); + assert_eq!(directory.join("control.sock").exists(), fail_at < 5); + assert_eq!( + retired_path(&directory, &owner, true).exists(), + fail_at >= 5 + ); + if journal.exists() { + let retained = fs::read(&journal).unwrap(); + assert!( + retire_recovered_publisher_locked_fenced( + &candidate, + &run, + &owner, + &"e".repeat(64), + None, + &lock, + &|| Ok(()), + ) + .is_err() + ); + assert_eq!(fs::read(&journal).unwrap(), retained); + assert_eq!(fs::read(&owner_path).unwrap(), before); + } + retire_recovered_publisher_locked_fenced( + &candidate, + &run, + &owner, + &receipt, + None, + &lock, + &|| Ok(()), + ) + .unwrap(); + assert!(!directory.join("control.sock").exists()); + assert!(!directory.join("owner.json").exists()); + assert_eq!( + fs::read(retired_path(&directory, &owner, false)).unwrap(), + before + ); + let archived_owner = fs::symlink_metadata(retired_path(&directory, &owner, false)).unwrap(); + let archived_socket = fs::symlink_metadata(retired_path(&directory, &owner, true)).unwrap(); + let journal_bytes = fs::read(&journal).unwrap(); + retire_recovered_publisher_locked_fenced( + &candidate, + &run, + &owner, + &receipt, + None, + &lock, + &|| Ok(()), + ) + .unwrap(); + assert_eq!(fs::read(&journal).unwrap(), journal_bytes); + assert_eq!( + id(&fs::symlink_metadata(retired_path(&directory, &owner, false)).unwrap()), + id(&archived_owner) + ); + assert_eq!( + id(&fs::symlink_metadata(retired_path(&directory, &owner, true)).unwrap()), + id(&archived_socket) + ); + assert_eq!( + fs::read(sibling_root.join("owner.json")).unwrap(), + sibling_before + ); + sibling.verify().unwrap(); + sibling.finish().unwrap(); + fs::remove_dir_all(sibling_root).unwrap(); + drop(lock); + fs::remove_dir_all(directory).unwrap(); + } +} + #[cfg(target_os = "macos")] #[test] fn retired_recovery_refuses_pending_missing_and_partial_proof_without_mutation() { diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index c54b3fb3b..d9343661d 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -56,8 +56,14 @@ mod bridge_recovery; mod bridges; pub use bridge_recovery::{export_bridge_recovery, inspect_bridge_recovery}; pub(in crate::provider) use bridges::{initialize_owner_registry, verify_owner_registry}; +#[cfg(target_os = "macos")] +mod acknowledged_publisher; mod cleanup_enrollment; #[cfg(target_os = "macos")] +pub use acknowledged_publisher::{ + AcknowledgedPublisherSelection, retire as retire_acknowledged_publisher, +}; +#[cfg(target_os = "macos")] mod dead_owner_cleanup; #[cfg(target_os = "macos")] mod live_owner_cleanup; From ff48ed717599673b22f39160c5bf6d947d2d7119 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 12:52:40 -0400 Subject: [PATCH 23/25] test: serialize the frontend acceptance stand-in's shared state Stand-in invocations ran concurrently (a foreground `up` while the driver polls `ps`) and each did an unlocked read-modify-write of state.json. A `ps` that loaded the state before the second `up` saved its foreground token then saved its stale copy over it, so `up` saw its token gone and exited 0 before readiness; lost call records failed the acceptance test as well. That failed 26 of 40 local runs, and Runtime state models on CI. Each invocation now holds an exclusive flock across its read-modify-write and releases it only before its long waits (the foreground loop and the ps-hang fault). 40 of 40 local runs pass. --- tests/python/test_native_frontend_acceptance.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/python/test_native_frontend_acceptance.py b/tests/python/test_native_frontend_acceptance.py index 5b1bf3710..09329f82d 100644 --- a/tests/python/test_native_frontend_acceptance.py +++ b/tests/python/test_native_frontend_acceptance.py @@ -22,11 +22,16 @@ # One stand-in serves as both hack-native and hack-v5 (by file name) over a shared state file. # `up` refuses unless a prepared base is required, then stays in the foreground until `down`. FAKE = r'''#!PYTHON -import json, os, pathlib, secrets, sys, time +import fcntl, json, os, pathlib, secrets, sys, time here = pathlib.Path(__file__).parent name = pathlib.Path(__file__).name path = here / "state.json" fault = os.environ.get("FAKE_FAULT", "") +# Invocations run concurrently (a foreground `up` while the driver polls `ps`). Each holds this +# exclusive lock across its read-modify-write of the shared state, so no invocation saves a stale +# copy over another's update; it is released only before a long wait. +lock = open(here / "state.lock", "a") +fcntl.flock(lock, fcntl.LOCK_EX) def load(): @@ -86,6 +91,7 @@ def reply(body, code=0): up_env={k: v for k, v in os.environ.items() if k.startswith("HACK_NATIVE_")}) state.setdefault("run", "r" + secrets.token_hex(8)) save(state) + fcntl.flock(lock, fcntl.LOCK_UN) deadline = time.time() + 30 while time.time() < deadline: time.sleep(0.02) @@ -94,6 +100,7 @@ def reply(body, code=0): sys.exit(3) if argv[0] == "ps": if fault == "ps-hang": + fcntl.flock(lock, fcntl.LOCK_UN) time.sleep(10) if state.get("foreground"): reply({"status": "observed", "phase": "ready-observed", "backend": "native", "run": state["run"], From df82fde82c658a39f5ecc571ee02dbd41cb2caf5 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 13:47:21 -0400 Subject: [PATCH 24/25] fix(runtime): release acknowledged dead dependency claims without losing evidence --- packages/runtime-core/README.md | 17 + packages/runtime-core/src/graph_cli.rs | 48 ++ .../src/graph_cli/restore_tests.rs | 40 ++ packages/runtime-core/src/main.rs | 1 + .../provider/graph/acknowledged_publisher.rs | 96 ++- .../src/provider/graph/dependency_slots.rs | 11 +- .../graph/dependency_slots/acknowledged.rs | 606 ++++++++++++++++++ .../provider/graph/foreground/transport.rs | 28 + .../runtime-core/src/provider/graph/mod.rs | 3 +- 9 files changed, 829 insertions(+), 21 deletions(-) create mode 100644 packages/runtime-core/src/provider/graph/dependency_slots/acknowledged.rs diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 07b75f8c8..5fe96325e 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -859,6 +859,23 @@ An old completed or cleanup-completed dependency journal is accepted as inert evidence only; malformed, incomplete or pending rebind state refuses at each effect boundary. Safe archival and the next restore remain separate checks. +If ordinary cleanup closed all dependency sockets but a later archival failure +left its reservation, first complete that exact publisher retirement. Inspect +`graph dependency-reservations --json`, then use `graph +release-acknowledged-dependencies --run-id RUN --expect-owner OWNER +--expect-receipt RECEIPT_SHA --expect-publisher PUBLISHER_SHA +--expect-reservation RESERVATION_SHA --json`. The final selector is the inspected +reservation fingerprint. This requires the current boot's acknowledged cleanup, +the exact completed publisher retirement, a dead reservation process matching +that publisher, the same run/Owner/boot and precisely the receipt's dependency +slots. Every selected socket must already be absent; even a matching stale socket +refuses. The selected record moves atomically without overwriting into the graph's +`dependency-reservation-retired-SHA.json` history, preserving bytes and inode. +Retry verifies that same record and proof. Changed selectors, another publisher, +pending/foreign records, occupied history and any reappearing socket refuse. +No socket, VM, graph receipt, dependency journal, data or sibling is removed. +This explicit recovery frees the claim; normal `up` still performs its own checks. + `restore-selection` then binds the witness's raw hash to the selected generation. The source checks use only an in-memory device projection; original retention and restore history consume the unchanged stopped receipt. The new receipt receives diff --git a/packages/runtime-core/src/graph_cli.rs b/packages/runtime-core/src/graph_cli.rs index 30e92adfa..d7e9c6406 100644 --- a/packages/runtime-core/src/graph_cli.rs +++ b/packages/runtime-core/src/graph_cli.rs @@ -25,6 +25,54 @@ pub fn command(candidate: &Candidate, args: &[&str]) -> Result (run, owner, receipt, publisher, reservation), + _ => return Err(invalid()), + }; + #[cfg(target_os = "macos")] + { + return graph::release_acknowledged_dependencies( + candidate, + graph::AcknowledgedPublisherSelection { + run, + owner, + receipt_sha256: receipt, + publisher_sha256: publisher, + }, + reservation, + ); + } + #[cfg(not(target_os = "macos"))] + { + let _ = (run, owner, receipt, publisher, reservation); + return Err(invalid()); + } + } if *action == "retire-acknowledged-publisher" { let (run, owner, receipt, publisher) = match *args { [ diff --git a/packages/runtime-core/src/graph_cli/restore_tests.rs b/packages/runtime-core/src/graph_cli/restore_tests.rs index 456fa6cce..d4f1d8df3 100644 --- a/packages/runtime-core/src/graph_cli/restore_tests.rs +++ b/packages/runtime-core/src/graph_cli/restore_tests.rs @@ -5,6 +5,46 @@ const RUN: &str = "11111111111111111111111111111111"; const PLAN: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; const GENERATION: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +#[test] +fn acknowledged_dependency_release_requires_all_selectors_and_forbids_other_effects() { + let fixture = Fixture::new(); + let candidate = Candidate::discover(&fixture.0).unwrap(); + let args = vec![ + "release-acknowledged-dependencies", + "--run-id", + RUN, + "--expect-owner", + RUN, + "--expect-receipt", + PLAN, + "--expect-publisher", + GENERATION, + "--expect-reservation", + PLAN, + ]; + for offset in [1, 3, 5, 7, 9] { + let mut missing = args.clone(); + missing.drain(offset..offset + 2); + assert_eq!( + command(&candidate, &missing).unwrap_err().code, + "graph_arguments" + ); + } + for effect in [ + "--remove-data", + "--environment-stdin", + "--accept-legacy-device-rebind", + ] { + let mut expanded = args.clone(); + expanded.push(effect); + assert_eq!( + command(&candidate, &expanded).unwrap_err().code, + "graph_arguments" + ); + } + assert!(!candidate.state_root.exists()); +} + #[test] fn acknowledged_retirement_requires_every_selector_and_forbids_other_effects() { let fixture = Fixture::new(); diff --git a/packages/runtime-core/src/main.rs b/packages/runtime-core/src/main.rs index 3066a79b2..b44244a3a 100644 --- a/packages/runtime-core/src/main.rs +++ b/packages/runtime-core/src/main.rs @@ -28,6 +28,7 @@ Usage: hack-local graph recover-live-owner --run-id <32-hex> --expect-receipt [--json] hack-local graph retire-recovered-publisher --run-id <32-hex> --expect-owner <32-hex> [--json] hack-local graph retire-acknowledged-publisher --run-id <32-hex> --expect-owner <32-hex> --expect-receipt <64-hex> --expect-publisher <64-hex> [--json] + hack-local graph release-acknowledged-dependencies --run-id <32-hex> --expect-owner <32-hex> --expect-receipt <64-hex> --expect-publisher <64-hex> --expect-reservation <64-hex> [--json] hack-local graph inspect-host-pin-recovery --run-id <32-hex> [--json] hack-local graph recover-host-pins --run-id <32-hex> --expect-selection <64-hex> --accept-legacy-device-rebind [--json] hack-local graph inspect-absent-publication-cleanup --run-id <32-hex> --original-owner-file --host-inspection-file [--json] diff --git a/packages/runtime-core/src/provider/graph/acknowledged_publisher.rs b/packages/runtime-core/src/provider/graph/acknowledged_publisher.rs index e47f77e5d..217a5ff53 100644 --- a/packages/runtime-core/src/provider/graph/acknowledged_publisher.rs +++ b/packages/runtime-core/src/provider/graph/acknowledged_publisher.rs @@ -95,6 +95,32 @@ fn acknowledged_effect(receipt: &Receipt, effect: [u8; 32]) -> Result<(), Candid Ok(()) } +fn verify_cleanup( + candidate: &Candidate, + engine: &Engine<'_>, + receipt: &Receipt, + root: &Path, + selected: &AcknowledgedPublisherSelection<'_>, +) -> Result<(), CandidateError> { + eligible(receipt, selected, engine.guest().boot_id())?; + exact_receipt(root, selected.receipt_sha256)?; + startup::require_dependency_rebind_complete(root, receipt)?; + let environment = environment::cleanup_inventory(candidate, engine, receipt, root)?; + let bridges = bridges::cleanup::read(engine, receipt, root)?; + acknowledged_effect( + receipt, + host_relay::cleanup_effect( + receipt, + engine.guest().boot_id(), + false, + &(&environment, &bridges), + )?, + )?; + host_relay::require_acknowledged_enrollment(receipt)?; + host_relay::inspect_cleanup(candidate, engine, receipt, false, &environment, &bridges)?; + engine.guest().verify() +} + /// Retire only a selected dead publication after independent current-boot ACK /// and guest absence checks. The existing immutable publisher journal preserves /// both original files and resumes either rename interruption. No graph receipt, @@ -117,27 +143,9 @@ pub fn retire( let lock = state::Lock::acquire_existing(&publication)?; host_pin_recovery::exact_lock_path(&publication, &lock)?; let (receipt, root) = load(candidate, &engine, selected.run)?; - eligible(&receipt, &selected, engine.guest().boot_id())?; - exact_receipt(&root, selected.receipt_sha256)?; - host_relay::require_acknowledged_enrollment(&receipt)?; - let environment = environment::cleanup_inventory(candidate, &engine, &receipt, &root)?; - let bridges = bridges::cleanup::read(&engine, &receipt, &root)?; let verify = || { host_pin_recovery::exact_lock_path(&publication, &lock)?; - exact_receipt(&root, selected.receipt_sha256)?; - startup::require_dependency_rebind_complete(&root, &receipt)?; - acknowledged_effect( - &receipt, - host_relay::cleanup_effect( - &receipt, - engine.guest().boot_id(), - false, - &(&environment, &bridges), - )?, - )?; - host_relay::require_acknowledged_enrollment(&receipt)?; - host_relay::inspect_cleanup(candidate, &engine, &receipt, false, &environment, &bridges)?; - engine.guest().verify() + verify_cleanup(candidate, &engine, &receipt, &root, &selected) }; verify()?; // This helper checks the selected publisher bytes, dead process, refused @@ -158,6 +166,56 @@ pub fn retire( ) } +/// Release a selected same-boot dependency claim only after exact publisher +/// retirement and current acknowledged cleanup. Every socket must be absent; +/// the record is exclusively moved to history, never deleted or overwritten. +pub fn release_dependencies( + candidate: &Candidate, + selected: AcknowledgedPublisherSelection<'_>, + expected_reservation: &str, +) -> Result { + if !hex(selected.run, 32) + || !hex(selected.owner, 32) + || !hex(selected.receipt_sha256, 64) + || !hex(selected.publisher_sha256, 64) + || !hex(expected_reservation, 64) + { + return Err(refused()); + } + let engine = Engine::connect_cleanup_wait(candidate)?; + let retired = + foreground::transport::Retired::acquire(candidate, selected.run)?.ok_or_else(refused)?; + let (receipt, root) = load(candidate, &engine, selected.run)?; + let verify = || { + retired.verify_recovery( + candidate, + selected.run, + selected.publisher_sha256, + selected.receipt_sha256, + )?; + verify_cleanup(candidate, &engine, &receipt, &root, &selected) + }; + verify()?; + let process = retired.publisher_process( + candidate, + selected.run, + selected.publisher_sha256, + selected.receipt_sha256, + )?; + dependency_slots::archive_acknowledged( + candidate, + &receipt, + engine.guest().boot_id(), + &process, + expected_reservation, + &verify, + )?; + verify()?; + Ok( + json!({"run":selected.run,"reservation_released":true,"record_retained":true,"data_retained":true,"same_boot":true}), + ) +} + #[cfg(test)] mod tests { use super::*; diff --git a/packages/runtime-core/src/provider/graph/dependency_slots.rs b/packages/runtime-core/src/provider/graph/dependency_slots.rs index 2fe5dfaa0..f4b2b016b 100644 --- a/packages/runtime-core/src/provider/graph/dependency_slots.rs +++ b/packages/runtime-core/src/provider/graph/dependency_slots.rs @@ -14,6 +14,8 @@ use std::{ path::{Path, PathBuf}, time::{Duration, Instant}, }; +mod acknowledged; +pub(super) use acknowledged::archive_acknowledged; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -51,6 +53,9 @@ fn read(path: &Path) -> Result { read_with_bytes(path).map(|(record, _)| record) } fn read_with_bytes(path: &Path) -> Result<(Record, Vec), CandidateError> { + read_with_identity(path).map(|(record, bytes, _)| (record, bytes)) +} +fn read_with_identity(path: &Path) -> Result<(Record, Vec, (u64, u64)), CandidateError> { let mut file = OpenOptions::new() .read(true) .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) @@ -74,8 +79,12 @@ fn read_with_bytes(path: &Path) -> Result<(Record, Vec), CandidateError> { if bytes.len() as u64 != m.len() { return Err(refused()); } + let current = fs::symlink_metadata(path).map_err(|_| refused())?; + if (current.dev(), current.ino()) != (m.dev(), m.ino()) || current.nlink() != 1 { + return Err(refused()); + } let record = serde_json::from_slice(&bytes).map_err(|_| refused())?; - Ok((record, bytes)) + Ok((record, bytes, (m.dev(), m.ino()))) } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/packages/runtime-core/src/provider/graph/dependency_slots/acknowledged.rs b/packages/runtime-core/src/provider/graph/dependency_slots/acknowledged.rs new file mode 100644 index 000000000..d62afffc1 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/dependency_slots/acknowledged.rs @@ -0,0 +1,606 @@ +//! A completed current-boot cleanup may strand a reservation if later archival +//! fails. Exact retirement and ACK authority come from the caller, not this file. +use super::*; +use std::ffi::CString; + +struct Paths<'a> { + directory: &'a Path, + home: &'a Path, + canonical_home: &'a Path, + history: &'a Path, +} + +struct HomePin<'a> { + alias: &'a Path, + target: &'a Path, + alias_id: (u64, u64), + target_id: (u64, u64), +} +impl<'a> HomePin<'a> { + fn capture(alias: &'a Path, target: &'a Path) -> Result { + let a = fs::symlink_metadata(alias).map_err(|_| refused())?; + let t = fs::symlink_metadata(target).map_err(|_| refused())?; + let pin = Self { + alias, + target, + alias_id: (a.dev(), a.ino()), + target_id: (t.dev(), t.ino()), + }; + pin.verify()?; + Ok(pin) + } + fn verify(&self) -> Result<(), CandidateError> { + let a = fs::symlink_metadata(self.alias).map_err(|_| refused())?; + let t = fs::symlink_metadata(self.target).map_err(|_| refused())?; + state::check_private_directory(self.target)?; + // SAFETY: geteuid has no parameters or side effects. + if !a.file_type().is_symlink() + || a.uid() != unsafe { libc::geteuid() } + || a.nlink() != 1 + || (a.dev(), a.ino()) != self.alias_id + || !t.is_dir() + || (t.dev(), t.ino()) != self.target_id + || fs::read_link(self.alias).map_err(|_| refused())? != self.target + { + return Err(refused()); + } + Ok(()) + } +} +struct Selection<'a> { + owner: &'a str, + run: &'a str, + boot: &'a str, + process: &'a ProcessIdentity, + slots: BTreeSet, + expected: &'a str, + capacity: u8, +} + +fn validate(record: &Record, selected: &Selection<'_>, home: &Path) -> Result<(), CandidateError> { + if !valid(record, selected.owner, selected.capacity) + || record.run != selected.run + || record.boot != selected.boot + || record.process != *selected.process + || record.slots.values().copied().collect::>() != selected.slots + || fingerprint(record)? != selected.expected + || identity::alive(record.process.pid)? + { + return Err(refused()); + } + // Even a matching stale socket is retained: this operation moves only the + // record after ordinary cleanup already closed every recorded listener. + for slot in record.slots.values() { + if !absent(&socket(home, *slot))? { + return Err(refused()); + } + } + Ok(()) +} + +fn rename_exclusive(source: &Path, destination: &Path) -> Result<(), CandidateError> { + let source = CString::new(source.as_os_str().as_encoded_bytes()).map_err(|_| refused())?; + let destination = + CString::new(destination.as_os_str().as_encoded_bytes()).map_err(|_| refused())?; + // SAFETY: both C strings remain valid during this macOS call. RENAME_EXCL + // atomically refuses an occupied target; no existing evidence is replaced. + if unsafe { libc::renamex_np(source.as_ptr(), destination.as_ptr(), libc::RENAME_EXCL) } != 0 { + return Err(refused()); + } + Ok(()) +} + +fn sync_directory(path: &Path, expected: (u64, u64)) -> Result<(), CandidateError> { + let file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK | libc::O_DIRECTORY) + .open(path) + .map_err(|_| refused())?; + let m = file.metadata().map_err(|_| refused())?; + if (m.dev(), m.ino()) != expected || !m.is_dir() { + return Err(refused()); + } + file.sync_all().map_err(state::io) +} + +fn archive( + paths: Paths<'_>, + selected: Selection<'_>, + verify: &dyn Fn() -> Result<(), CandidateError>, +) -> Result<(), CandidateError> { + verify()?; + state::check_private_directory(paths.directory)?; + state::check_private_directory(paths.history)?; + let home = HomePin::capture(paths.home, paths.canonical_home)?; + let directory_metadata = fs::symlink_metadata(paths.directory).map_err(|_| refused())?; + let history_metadata = fs::symlink_metadata(paths.history).map_err(|_| refused())?; + let parents = [ + (directory_metadata.dev(), directory_metadata.ino()), + (history_metadata.dev(), history_metadata.ino()), + ]; + let verify_paths = || { + home.verify()?; + for (path, expected) in [paths.directory, paths.history].into_iter().zip(parents) { + state::check_private_directory(path)?; + let m = fs::symlink_metadata(path).map_err(|_| refused())?; + if (m.dev(), m.ino()) != expected { + return Err(refused()); + } + } + Ok(()) + }; + records(paths.directory, selected.owner, selected.capacity)?; + let source = paths.directory.join(format!("{}.json", selected.run)); + let target = paths.history.join(format!( + "dependency-reservation-retired-{}.json", + selected.expected + )); + if !absent(&source.with_extension("pending"))? || !absent(&target.with_extension("pending"))? { + return Err(refused()); + } + if absent(&source)? { + // A retry must find the exact retained record; ordinary absence is not + // authority and a new same-run reservation is never adopted. + let (record, bytes, id) = read_with_identity(&target)?; + verify_paths()?; + validate(&record, &selected, paths.canonical_home)?; + verify()?; + verify_paths()?; + for (path, expected) in [paths.directory, paths.history].into_iter().zip(parents) { + sync_directory(path, expected)?; + } + let (again, current, current_id) = read_with_identity(&target)?; + if again != record || current != bytes || current_id != id || !absent(&source)? { + return Err(refused()); + } + validate(&again, &selected, paths.canonical_home)?; + return Ok(()); + } + if !absent(&target)? { + return Err(refused()); + } + let (record, bytes, id) = read_with_identity(&source)?; + verify_paths()?; + validate(&record, &selected, paths.canonical_home)?; + verify()?; + let (again, current, current_id) = read_with_identity(&source)?; + verify_paths()?; + if again != record || current != bytes || current_id != id { + return Err(refused()); + } + validate(&again, &selected, paths.canonical_home)?; + let file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(&source) + .map_err(|_| refused())?; + let m = file.metadata().map_err(|_| refused())?; + if !m.is_file() || (m.dev(), m.ino()) != id || m.nlink() != 1 { + return Err(refused()); + } + file.sync_all().map_err(state::io)?; + verify()?; + // The provider/publication locks stay held. Recheck the selected inode after + // the final external fence, including tests that replace it at that window. + let (again, current, current_id) = read_with_identity(&source)?; + verify_paths()?; + if again != record || current != bytes || current_id != id { + return Err(refused()); + } + validate(&again, &selected, paths.canonical_home)?; + rename_exclusive(&source, &target)?; + sync_directory(paths.history, parents[1])?; + sync_directory(paths.directory, parents[0])?; + let (retained, retained_bytes, retained_id) = read_with_identity(&target)?; + verify_paths()?; + if retained != record || retained_bytes != bytes || retained_id != id || !absent(&source)? { + return Err(refused()); + } + validate(&retained, &selected, paths.canonical_home)?; + verify()?; + verify_paths()?; + let (again, current, current_id) = read_with_identity(&target)?; + if again != record || current != bytes || current_id != id || !absent(&source)? { + return Err(refused()); + } + validate(&again, &selected, paths.canonical_home) +} + +/// Caller holds the provider and retired-publisher locks and supplies an exact +/// current ACK fence. No socket, graph receipt, named data or sibling is changed. +pub(in crate::provider::graph) fn archive_acknowledged( + candidate: &Candidate, + receipt: &Receipt, + boot: &str, + process: &ProcessIdentity, + expected: &str, + verify: &dyn Fn() -> Result<(), CandidateError>, +) -> Result<(), CandidateError> { + let owner = state::Owner::load(candidate)?; + if receipt.phase != "stopped-data-retained" || receipt.owner != owner.token { + return Err(refused()); + } + let directory = root(candidate); + let history = super::super::directory(candidate, &receipt.run)?; + let canonical_home = candidate.state_root.join("run/smolvm/home"); + let slots = receipt + .relay_startup + .as_ref() + .ok_or_else(refused)? + .services + .values() + .flat_map(|service| service.bindings.values().map(|binding| binding.slot)) + .collect(); + archive( + Paths { + directory: &directory, + home: &owner.short_home, + canonical_home: &canonical_home, + history: &history, + }, + Selection { + owner: &owner.token, + run: &receipt.run, + boot, + process, + slots, + expected, + capacity: owner.dependency_sockets.ok_or_else(refused)?.slots, + }, + verify, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{cell::Cell, os::unix::fs::PermissionsExt, process::Command}; + + struct Fixture { + _root: super::super::super::tests::Fixture, + directory: PathBuf, + home: PathBuf, + canonical_home: PathBuf, + history: PathBuf, + record: Record, + expected: String, + } + impl Fixture { + fn new() -> Self { + let root = super::super::super::tests::Fixture::new(); + let directory = root.0.join("assignments"); + let canonical_home = root.0.join("home"); + let home = PathBuf::from(format!( + "/private/tmp/hkdr-{}", + &format!( + "{:x}", + Sha256::digest(root.0.as_os_str().as_encoded_bytes()) + )[..16] + )); + let history = root.0.join("graph"); + for path in [&directory, &canonical_home, &history] { + state::private_directory(path).unwrap(); + } + std::os::unix::fs::symlink(&canonical_home, &home).unwrap(); + let mut child = Command::new("/bin/sleep").arg("30").spawn().unwrap(); + let process = identity::observe(child.id() as i32).unwrap(); + child.kill().unwrap(); + child.wait().unwrap(); + let record = Record { + version: 1, + owner: "a".repeat(32), + boot: "b".repeat(36), + run: "1".repeat(32), + token: "c".repeat(32), + process, + slots: BTreeMap::from([(0, 0)]), + sockets: BTreeMap::from([(0, (1, 2))]), + }; + state::write(&directory.join(format!("{}.json", record.run)), &record).unwrap(); + let expected = fingerprint(&record).unwrap(); + Self { + _root: root, + directory, + home, + canonical_home, + history, + record, + expected, + } + } + fn source(&self) -> PathBuf { + self.directory.join(format!("{}.json", self.record.run)) + } + fn target(&self) -> PathBuf { + self.history.join(format!( + "dependency-reservation-retired-{}.json", + self.expected + )) + } + fn archive( + &self, + verify: &dyn Fn() -> Result<(), CandidateError>, + ) -> Result<(), CandidateError> { + archive( + Paths { + directory: &self.directory, + home: &self.home, + canonical_home: &self.canonical_home, + history: &self.history, + }, + Selection { + owner: &self.record.owner, + run: &self.record.run, + boot: &self.record.boot, + process: &self.record.process, + slots: BTreeSet::from([0]), + expected: &self.expected, + capacity: 2, + }, + verify, + ) + } + } + impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_file(&self.home).unwrap(); + } + } + + #[test] + fn exact_dead_claim_is_retained_with_same_bytes_inode_and_sibling() { + let fixture = Fixture::new(); + let before = fs::read(fixture.source()).unwrap(); + let m = fs::symlink_metadata(fixture.source()).unwrap(); + let mut sibling = fixture.record.clone(); + sibling.run = "2".repeat(32); + sibling.slots = BTreeMap::from([(0, 1)]); + sibling.sockets = BTreeMap::from([(1, (1, 3))]); + let path = fixture.directory.join(format!("{}.json", sibling.run)); + state::write(&path, &sibling).unwrap(); + let sibling_before = fs::read(&path).unwrap(); + fixture.archive(&|| Ok(())).unwrap(); + assert!(!fixture.source().exists()); + assert_eq!(fs::read(fixture.target()).unwrap(), before); + let retained = fs::symlink_metadata(fixture.target()).unwrap(); + assert_eq!((retained.dev(), retained.ino()), (m.dev(), m.ino())); + assert_eq!(fs::read(path).unwrap(), sibling_before); + fixture.archive(&|| Ok(())).unwrap(); + } + + #[test] + fn mismatched_process_boot_slots_and_live_owner_refuse() { + let fixture = Fixture::new(); + let base = &fixture.record; + let selected = Selection { + owner: &base.owner, + run: &base.run, + boot: &base.boot, + process: &base.process, + slots: BTreeSet::from([0]), + expected: &fixture.expected, + capacity: 2, + }; + for change in ["process", "boot", "slots", "run", "token", "live"] { + let mut changed = base.clone(); + match change { + "process" => changed.process.start_micros += 1, + "boot" => changed.boot = "d".repeat(36), + "slots" => changed.slots.insert(0, 1).map(|_| ()).unwrap(), + "run" => changed.run = "9".repeat(32), + "token" => changed.token = "9".repeat(32), + "live" => changed.process = identity::observe(std::process::id() as i32).unwrap(), + _ => unreachable!(), + } + assert!( + validate(&changed, &selected, &fixture.home).is_err(), + "{change}" + ); + } + let live = identity::observe(std::process::id() as i32).unwrap(); + let mut changed = base.clone(); + changed.process = live.clone(); + let expected = fingerprint(&changed).unwrap(); + let live_selection = Selection { + process: &live, + expected: &expected, + ..selected + }; + assert!(validate(&changed, &live_selection, &fixture.home).is_err()); + assert!(fixture.source().exists()); + } + + #[test] + fn every_socket_pending_record_and_occupied_history_is_preserved() { + use std::os::unix::net::UnixListener; + for reason in ["socket", "pending", "history"] { + let fixture = Fixture::new(); + let before = fs::read(fixture.source()).unwrap(); + let path = match reason { + "socket" => socket(&fixture.home, 0), + "pending" => fixture.source().with_extension("pending"), + "history" => fixture.target(), + _ => unreachable!(), + }; + let listener = if reason == "socket" { + Some(UnixListener::bind(&path).unwrap()) + } else { + fs::write(&path, b"foreign or partial").unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + None + }; + let id = fs::symlink_metadata(&path).unwrap(); + assert!(fixture.archive(&|| Ok(())).is_err(), "{reason}"); + assert_eq!(fs::read(fixture.source()).unwrap(), before); + let after = fs::symlink_metadata(path).unwrap(); + assert_eq!((id.dev(), id.ino()), (after.dev(), after.ino())); + drop(listener); + } + } + + #[test] + fn failed_fences_preserve_claim_or_allow_exact_post_move_retry() { + for fault in 1..=4 { + let fixture = Fixture::new(); + let before = fs::read(fixture.source()).unwrap(); + let calls = Cell::new(0); + assert!( + fixture + .archive(&|| { + calls.set(calls.get() + 1); + if calls.get() == fault { + Err(refused()) + } else { + Ok(()) + } + }) + .is_err(), + "fence {fault}" + ); + if fault < 4 { + assert_eq!(fs::read(fixture.source()).unwrap(), before); + assert!(!fixture.target().exists()); + } else { + assert!(!fixture.source().exists()); + assert_eq!(fs::read(fixture.target()).unwrap(), before); + } + fixture.archive(&|| Ok(())).unwrap(); + assert_eq!(fs::read(fixture.target()).unwrap(), before); + } + } + + #[test] + fn same_bytes_replacement_and_late_history_collision_refuse() { + for replacement in ["source", "history", "socket"] { + let fixture = Fixture::new(); + let before = fs::read(fixture.source()).unwrap(); + let calls = Cell::new(0); + let displaced = fixture.history.join("displaced.json"); + assert!( + fixture + .archive(&|| { + calls.set(calls.get() + 1); + if calls.get() == 3 { + match replacement { + "source" => { + fs::rename(fixture.source(), &displaced).unwrap(); + fs::write(fixture.source(), &before).unwrap(); + fs::set_permissions( + fixture.source(), + fs::Permissions::from_mode(0o600), + ) + .unwrap(); + } + "history" => { + fs::write(fixture.target(), b"foreign").unwrap(); + } + "socket" => { + fs::write(socket(&fixture.home, 0), b"foreign").unwrap(); + } + _ => unreachable!(), + } + } + Ok(()) + }) + .is_err(), + "{replacement}" + ); + assert_eq!(fs::read(fixture.source()).unwrap(), before); + if replacement == "history" { + assert_eq!(fs::read(fixture.target()).unwrap(), b"foreign"); + } + if replacement == "source" { + assert_eq!(fs::read(displaced).unwrap(), before); + } + if replacement == "socket" { + assert_eq!(fs::read(socket(&fixture.home, 0)).unwrap(), b"foreign"); + } + } + } + + #[test] + fn alias_home_and_reservation_parent_replacement_refuse_before_release() { + for replacement in ["alias", "home", "assignments", "history"] { + let fixture = Fixture::new(); + let before = fs::read(fixture.source()).unwrap(); + let calls = Cell::new(0); + let displaced = fixture._root.0.join("displaced"); + assert!( + fixture + .archive(&|| { + calls.set(calls.get() + 1); + if calls.get() == 3 { + match replacement { + "alias" => { + fs::rename(&fixture.home, &displaced).unwrap(); + std::os::unix::fs::symlink( + &fixture.canonical_home, + &fixture.home, + ) + .unwrap(); + } + "home" => { + fs::rename(&fixture.canonical_home, &displaced).unwrap(); + state::private_directory(&fixture.canonical_home).unwrap(); + } + "assignments" => { + fs::rename(&fixture.directory, &displaced).unwrap(); + state::private_directory(&fixture.directory).unwrap(); + fs::write(fixture.source(), &before).unwrap(); + fs::set_permissions( + fixture.source(), + fs::Permissions::from_mode(0o600), + ) + .unwrap(); + } + "history" => { + fs::rename(&fixture.history, &displaced).unwrap(); + state::private_directory(&fixture.history).unwrap(); + } + _ => unreachable!(), + } + } + Ok(()) + }) + .is_err(), + "{replacement}" + ); + assert_eq!(fs::read(fixture.source()).unwrap(), before); + assert!(!fixture.target().exists()); + } + } + + #[test] + fn retry_and_final_success_recheck_retained_record_after_external_fence() { + for retry in [false, true] { + let fixture = Fixture::new(); + let before = fs::read(fixture.source()).unwrap(); + if retry { + fixture.archive(&|| Ok(())).unwrap(); + } + let calls = Cell::new(0); + let displaced = fixture.history.join("original.json"); + assert!( + fixture + .archive(&|| { + calls.set(calls.get() + 1); + if calls.get() == if retry { 2 } else { 4 } { + fs::rename(fixture.target(), &displaced).unwrap(); + fs::write(fixture.target(), &before).unwrap(); + fs::set_permissions( + fixture.target(), + fs::Permissions::from_mode(0o600), + ) + .unwrap(); + } + Ok(()) + }) + .is_err(), + "retry={retry}" + ); + assert_eq!(fs::read(fixture.target()).unwrap(), before); + assert_eq!(fs::read(displaced).unwrap(), before); + assert!(!fixture.source().exists()); + } + } +} diff --git a/packages/runtime-core/src/provider/graph/foreground/transport.rs b/packages/runtime-core/src/provider/graph/foreground/transport.rs index 42b3205ac..5e43651d3 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport.rs @@ -1082,6 +1082,34 @@ pub(in crate::provider::graph) struct Retired { _lock: state::Lock, } impl Retired { + /// Obtain the process identity only from the exact completed retirement. + /// This does not infer ownership from pathname absence or PID death. + pub fn publisher_process( + &self, + candidate: &Candidate, + run: &str, + owner_sha256: &str, + complete_sha256: &str, + ) -> Result { + self.verify_recovery(candidate, run, owner_sha256, complete_sha256)?; + let intent: Retirement = state::read(&retirement_path(&self.root, owner_sha256)) + .map_err(|_| retirement_refused())?; + let (socket_original, record_original) = verify_retirement( + candidate, + run, + (owner_sha256, complete_sha256), + &self.root, + &self._lock, + &intent, + None, + )?; + if socket_original || record_original { + return Err(retirement_refused()); + } + self.verify()?; + Ok(intent.owner.process) + } + pub fn acquire(candidate: &Candidate, run: &str) -> Result, CandidateError> { let root = root(candidate, run)?; state::check_private_directory(&root)?; diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index d9343661d..7d53ab86b 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -61,7 +61,8 @@ mod acknowledged_publisher; mod cleanup_enrollment; #[cfg(target_os = "macos")] pub use acknowledged_publisher::{ - AcknowledgedPublisherSelection, retire as retire_acknowledged_publisher, + AcknowledgedPublisherSelection, release_dependencies as release_acknowledged_dependencies, + retire as retire_acknowledged_publisher, }; #[cfg(target_os = "macos")] mod dead_owner_cleanup; From b49fbdae1111f0f5b3db74f871bd44e807550c2d Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 13:54:14 -0400 Subject: [PATCH 25/25] refactor(runtime): name the observed dependency record tuple --- packages/runtime-core/src/provider/graph/dependency_slots.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/runtime-core/src/provider/graph/dependency_slots.rs b/packages/runtime-core/src/provider/graph/dependency_slots.rs index f4b2b016b..45cf82a50 100644 --- a/packages/runtime-core/src/provider/graph/dependency_slots.rs +++ b/packages/runtime-core/src/provider/graph/dependency_slots.rs @@ -55,7 +55,9 @@ fn read(path: &Path) -> Result { fn read_with_bytes(path: &Path) -> Result<(Record, Vec), CandidateError> { read_with_identity(path).map(|(record, bytes, _)| (record, bytes)) } -fn read_with_identity(path: &Path) -> Result<(Record, Vec, (u64, u64)), CandidateError> { +/// Parsed record, original bytes and the no-follow descriptor's file identity. +type ObservedRecord = (Record, Vec, (u64, u64)); +fn read_with_identity(path: &Path) -> Result { let mut file = OpenOptions::new() .read(true) .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK)