From 10697ae899e5b3b26142581787250449448cb6ae Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Tue, 29 Sep 2026 21:41:39 -0400 Subject: [PATCH 1/9] fix: recover missing native provider HOME aliases --- docs/guides/native-candidate.md | 15 + .../runtime-core/src/provider/lifecycle.rs | 25 +- .../src/provider/lifecycle/short_home.rs | 70 +++++ .../provider/lifecycle/short_home/tests.rs | 265 ++++++++++++++++++ packages/runtime-core/src/provider/state.rs | 88 +++++- 5 files changed, 455 insertions(+), 8 deletions(-) create mode 100644 packages/runtime-core/src/provider/lifecycle/short_home.rs create mode 100644 packages/runtime-core/src/provider/lifecycle/short_home/tests.rs diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 7a31e89b0..2dbe382d3 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -118,6 +118,21 @@ complete bundle to change the candidate. binary refusing newer state is an unsupported downgrade, not a successful rollback. Preserve that state; do not rewrite receipts or adopt another home's data. +Host cleanup or a reboot can remove the temporary `/private/tmp/hkl-` HOME +alias while the private candidate home and disks remain intact. Normal commands +report `provider_home_missing`; they do not initialize another pool or recreate the +alias during observation. Explicit `runtime recover --json` can restore only that +absent exact alias, after verifying the receipt-bound dead provider, both recorded +disks, free VM lock, closed disk handles, and no active provider command. Existing +files, directories, foreign links, a live or reused PID, and uncertain ownership +refuse without replacement. Recovery rechecks the receipt before exclusive creation, +then validates socket absence and flushes the disks before recording +`recovered-unclean`. If those final checks fail, it reports +`provider_home_restored_recovery_incomplete`, keeps the exact owned alias and retained +data, and requires inspection before retry. No guest is started by alias recovery. +Interrupted starts without a recorded process or both identified disks remain +outside this repair path. + Compatibility is qualified for specific bundle hashes and state formats. The displayed version alone does not establish frontend/executor or downgrade compatibility. V4 and candidate homes remain separate; this procedure does not diff --git a/packages/runtime-core/src/provider/lifecycle.rs b/packages/runtime-core/src/provider/lifecycle.rs index 60e7109d3..8b47b97b0 100644 --- a/packages/runtime-core/src/provider/lifecycle.rs +++ b/packages/runtime-core/src/provider/lifecycle.rs @@ -3,6 +3,7 @@ mod prepared_boot; #[cfg(any(target_os = "macos", test))] mod private_child; mod relay_process; +mod short_home; use super::{ admission, agent, artifact, identity, process, state::{self, Owner, io}, @@ -1812,6 +1813,12 @@ fn finish_absent( value: &str, record_disks: bool, ) -> Result<(), CandidateError> { + let vm_lock = lock_absent_disks(candidate, owner)?; + finish_absent_locked(candidate, owner, value, record_disks, &vm_lock) +} + +/// Keep this descriptor through alias restoration and the stopped receipt. +fn lock_absent_disks(candidate: &Candidate, owner: &Owner) -> Result { let directory = owner.real_data_dir(candidate)?; let vm_lock = OpenOptions::new() .read(true) @@ -1847,6 +1854,16 @@ fn finish_absent( )); } verify_disks(candidate, owner)?; + Ok(vm_lock) +} + +fn finish_absent_locked( + candidate: &Candidate, + owner: &mut Owner, + value: &str, + record_disks: bool, + _vm_lock: &File, +) -> Result<(), CandidateError> { if record_disks && owner.storage.is_none() { owner.storage = Some(identity::disk( &owner.real_data_dir(candidate)?.join("storage.raw"), @@ -1896,7 +1913,13 @@ fn finish_absent( } pub fn recover(candidate: &Candidate) -> Result { - let initial = status(candidate)?; + let initial = match status(candidate) { + Ok(initial) => initial, + Err(error) if error.code == "provider_home_missing" => { + return short_home::recover(candidate); + } + Err(error) => return Err(error), + }; if initial.phase == "uninitialized" { return Ok(initial); } diff --git a/packages/runtime-core/src/provider/lifecycle/short_home.rs b/packages/runtime-core/src/provider/lifecycle/short_home.rs new file mode 100644 index 000000000..6678ab602 --- /dev/null +++ b/packages/runtime-core/src/provider/lifecycle/short_home.rs @@ -0,0 +1,70 @@ +//! Explicit two-phase recovery of the receipt-bound temporary HOME after host cleanup. +//! +//! No provider is launched and no existing alias is replaced. Durable disks and VM absence +//! are audited before restoring the short socket path; socket absence is then checked before +//! committing the recovered phase. A later refusal retains the exact alias and unchanged data. +use super::{ + Owner, RuntimeStatus, binary, finish_absent_locked, identity, lock_absent_disks, root, state, + status, verify_disks, +}; +use crate::{Candidate, CandidateError}; + +fn dead_provider(candidate: &Candidate, owner: &Owner) -> Result<(), CandidateError> { + let process = owner.process.as_ref().ok_or_else(|| { + CandidateError::new( + "recovery_required", + "Missing HOME recovery requires a recorded provider identity; nothing was changed.", + ) + })?; + // SAFETY: geteuid has no preconditions. + identity::verify(process, process, &binary(candidate), unsafe { + libc::geteuid() + })?; + if identity::alive(process.pid)? || identity::executable_running(&binary(candidate))? { + return Err(CandidateError::new( + "recovery_required", + "Missing HOME recovery requires a confirmed dead provider and no active provider command; nothing was changed.", + )); + } + if !owner.created || owner.storage.is_none() || owner.overlay.is_none() { + return Err(CandidateError::new( + "recovery_required", + "Missing HOME recovery requires both previously identified disks; nothing was adopted.", + )); + } + Ok(()) +} + +pub(super) fn recover(candidate: &Candidate) -> Result { + let _operation = state::Lock::acquire_existing(&root(candidate))?; + let mut owner = Owner::load_for_short_home_recovery(candidate)?; + dead_provider(candidate, &owner)?; + let vm_lock = lock_absent_disks(candidate, &owner)?; + // Recheck immediately before the only new external effect. The locks fence cooperative + // writers; no signal, disk adoption, overwrite or receipt update happens in this phase. + dead_provider(candidate, &owner)?; + verify_disks(candidate, &owner)?; + owner.restore_missing_short_home(candidate)?; + let mut finish = || -> Result<(), CandidateError> { + let observed = Owner::load(candidate)?; + if observed != owner { + return Err(CandidateError::new( + "foreign_state", + "Provider receipt changed after HOME restoration.", + )); + } + dead_provider(candidate, &owner)?; + verify_disks(candidate, &owner)?; + finish_absent_locked(candidate, &mut owner, "recovered-unclean", false, &vm_lock) + }; + finish().map_err(|error| { + CandidateError::new( + "provider_home_restored_recovery_incomplete", + format!("Owned HOME alias restored; runtime recovery is incomplete ({}). Data retained; inspect and retry runtime recover.", error.code), + ) + })?; + status(candidate) +} + +#[cfg(all(test, target_os = "macos"))] +mod tests; diff --git a/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs b/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs new file mode 100644 index 000000000..f22b62192 --- /dev/null +++ b/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs @@ -0,0 +1,265 @@ +use super::*; +use crate::provider::{NetworkIntent, Profile}; +use std::{ + fs, + os::{ + fd::AsRawFd, + unix::{fs::MetadataExt, net::UnixListener}, + }, + path::PathBuf, + process::Command, +}; + +struct Pool { + candidate: Candidate, + owner: Owner, + directory: PathBuf, +} +impl Pool { + fn new() -> Self { + let mut child = Command::new("/bin/sleep").arg("30").spawn().unwrap(); + let mut process = identity::observe(child.id() as i32).unwrap(); + child.kill().unwrap(); + child.wait().unwrap(); + let directory = fs::canonicalize(std::env::temp_dir()) + .unwrap() + .join(format!( + "hack-home-recovery-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + state::private_directory(&directory).unwrap(); + let candidate = Candidate::discover(&directory).unwrap(); + let operation = state::Lock::acquire(&root(&candidate)).unwrap(); + let mut owner = Owner::create( + &candidate, + Profile::Development, + None, + NetworkIntent::Isolated, + ) + .unwrap(); + let data = owner.real_data_dir(&candidate).unwrap(); + state::private_directory(&data).unwrap(); + for (name, tag) in [("storage.raw", 1u8), ("overlay.raw", 2)] { + let mut bytes = vec![0u8; 4096]; + bytes[1080..1082].copy_from_slice(&[0x53, 0xef]); + bytes[1128..1144].fill(tag); + bytes[2048..2059].copy_from_slice(b"data-marker"); + fs::write(data.join(name), bytes).unwrap(); + } + fs::write(data.join("vm.lock"), b"").unwrap(); + fs::write(data.join("name"), &owner.machine).unwrap(); + process.executable = binary(&candidate); + owner.process = Some(process); + owner.created = true; + owner.phase = "running".into(); + owner.storage = Some(identity::disk(&data.join("storage.raw")).unwrap()); + owner.overlay = Some(identity::disk(&data.join("overlay.raw")).unwrap()); + owner.save(&candidate).unwrap(); + drop(operation); + Self { + candidate, + owner, + directory, + } + } + fn remove_alias(&self) { + fs::remove_file(&self.owner.short_home).unwrap(); + } + fn receipt(&self) -> Vec { + fs::read(root(&self.candidate).join("owner.json")).unwrap() + } + fn data(&self) -> PathBuf { + self.owner.real_data_dir(&self.candidate).unwrap() + } + fn unchanged(&self, before: &[u8]) { + assert_eq!(self.receipt(), before); + assert!(self.owner.short_home.symlink_metadata().is_err()); + } +} +impl Drop for Pool { + fn drop(&mut self) { + let _ = fs::remove_file(&self.owner.short_home); + fs::remove_dir_all(&self.directory).unwrap(); + } +} + +#[test] +fn explicit_recovery_restores_only_missing_alias_and_preserves_disk_bytes() { + let pool = Pool::new(); + let storage = fs::read(pool.data().join("storage.raw")).unwrap(); + let overlay = fs::read(pool.data().join("overlay.raw")).unwrap(); + pool.remove_alias(); + assert_eq!( + status(&pool.candidate).unwrap_err().code, + "provider_home_missing" + ); + let recovered = crate::provider::recover(&pool.candidate).unwrap(); + assert_eq!(recovered.phase, "recovered-unclean"); + assert_eq!(recovered.process_alive, Some(false)); + assert_eq!( + fs::read_link(&pool.owner.short_home).unwrap(), + root(&pool.candidate).join("home") + ); + assert_eq!(fs::read(pool.data().join("storage.raw")).unwrap(), storage); + assert_eq!(fs::read(pool.data().join("overlay.raw")).unwrap(), overlay); + assert_eq!( + Owner::load(&pool.candidate).unwrap().storage, + pool.owner.storage + ); +} + +#[test] +fn live_or_reused_pid_and_missing_disk_identity_refuse_without_alias_effect() { + for case in 0..3 { + let mut pool = Pool::new(); + if case < 2 { + let p = pool.owner.process.as_mut().unwrap(); + p.pid = std::process::id() as i32; + p.start_micros = if case == 0 { + identity::observe(p.pid).unwrap().start_micros + } else { + 1 + }; + } else { + pool.owner.overlay = None; + } + pool.owner.save(&pool.candidate).unwrap(); + pool.remove_alias(); + let before = pool.receipt(); + assert_eq!( + crate::provider::recover(&pool.candidate).unwrap_err().code, + "recovery_required" + ); + pool.unchanged(&before); + } +} + +#[test] +fn changed_disk_held_vm_lock_and_open_disk_refuse_before_alias_creation() { + for case in 0..3 { + let pool = Pool::new(); + pool.remove_alias(); + let before = pool.receipt(); + let mut held = None; + if case == 0 { + fs::write(pool.data().join("overlay.raw"), b"changed disk").unwrap(); + } + if case == 1 { + let f = fs::File::open(pool.data().join("vm.lock")).unwrap(); + assert_eq!( + unsafe { libc::flock(f.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }, + 0 + ); + held = Some(f); + } + if case == 2 { + held = Some(fs::File::open(pool.data().join("storage.raw")).unwrap()); + } + assert!(crate::provider::recover(&pool.candidate).is_err()); + pool.unchanged(&before); + drop(held); + } +} + +#[test] +fn existing_file_directory_and_foreign_symlink_are_never_replaced() { + for case in 0..3 { + let pool = Pool::new(); + pool.remove_alias(); + let before = pool.receipt(); + if case == 0 { + fs::write(&pool.owner.short_home, b"foreign marker").unwrap(); + } + if case == 1 { + fs::create_dir(&pool.owner.short_home).unwrap(); + } + if case == 2 { + std::os::unix::fs::symlink(&pool.directory, &pool.owner.short_home).unwrap(); + } + let inode = fs::symlink_metadata(&pool.owner.short_home).unwrap().ino(); + assert_eq!( + crate::provider::recover(&pool.candidate).unwrap_err().code, + "foreign_state" + ); + assert_eq!(pool.receipt(), before); + assert_eq!( + fs::symlink_metadata(&pool.owner.short_home).unwrap().ino(), + inode + ); + if case == 1 { + fs::remove_dir(&pool.owner.short_home).unwrap(); + } + } +} + +#[test] +fn active_socket_leaves_explicit_incomplete_recovery_then_retry_finishes() { + let pool = Pool::new(); + let listener = UnixListener::bind(pool.owner.data_dir().join("agent.sock")).unwrap(); + pool.remove_alias(); + let before = pool.receipt(); + let error = crate::provider::recover(&pool.candidate).unwrap_err(); + assert_eq!(error.code, "provider_home_restored_recovery_incomplete"); + assert!(error.message.contains("stop_uncertain")); + assert_eq!(pool.receipt(), before); + assert_eq!( + fs::read_link(&pool.owner.short_home).unwrap(), + root(&pool.candidate).join("home") + ); + drop(listener); + assert_eq!( + crate::provider::recover(&pool.candidate).unwrap().phase, + "recovered-unclean" + ); +} + +#[test] +fn receipt_change_and_concurrent_alias_creation_refuse_exclusive_repair() { + let mut pool = Pool::new(); + pool.remove_alias(); + let observed = Owner::load_for_short_home_recovery(&pool.candidate).unwrap(); + pool.owner.phase = "stopped".into(); + pool.owner.save(&pool.candidate).unwrap(); + let before = pool.receipt(); + assert_eq!( + observed + .restore_missing_short_home(&pool.candidate) + .unwrap_err() + .code, + "foreign_state" + ); + pool.unchanged(&before); + std::os::unix::fs::symlink(root(&pool.candidate).join("home"), &pool.owner.short_home).unwrap(); + let inode = fs::symlink_metadata(&pool.owner.short_home).unwrap().ino(); + assert_eq!( + pool.owner + .restore_missing_short_home(&pool.candidate) + .unwrap_err() + .code, + "socket_alias_collision" + ); + assert_eq!( + fs::symlink_metadata(&pool.owner.short_home).unwrap().ino(), + inode + ); + assert_eq!(pool.receipt(), before); +} + +#[test] +fn pending_owner_update_is_preserved_and_refuses_alias_repair() { + let pool = Pool::new(); + pool.remove_alias(); + let before = pool.receipt(); + let pending = root(&pool.candidate).join("owner.pending"); + fs::write(&pending, b"interrupted update").unwrap(); + assert_eq!( + crate::provider::recover(&pool.candidate).unwrap_err().code, + "recovery_required" + ); + pool.unchanged(&before); + assert_eq!(fs::read(&pending).unwrap(), b"interrupted update"); +} diff --git a/packages/runtime-core/src/provider/state.rs b/packages/runtime-core/src/provider/state.rs index 7f3b1a81a..e8810dbe1 100644 --- a/packages/runtime-core/src/provider/state.rs +++ b/packages/runtime-core/src/provider/state.rs @@ -57,7 +57,6 @@ impl Lock { } /// Observe existing state without initializing a directory or lock file. - #[cfg(target_os = "macos")] pub fn acquire_existing(root: &Path) -> Result { check_private_directory(root)?; let file = OpenOptions::new() @@ -177,7 +176,7 @@ impl Default for ReclamationPolicy { } } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Owner { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -211,8 +210,34 @@ pub struct Owner { } impl Owner { pub fn load(candidate: &Candidate) -> Result { + Self::load_with_short_home(candidate, false) + } + + /// Explicit recovery may inspect a missing temporary alias, never a replaced one. + /// Loading here is read-only; the lifecycle boundary still proves provider absence. + pub(super) fn load_for_short_home_recovery( + candidate: &Candidate, + ) -> Result { + Self::load_with_short_home(candidate, true) + } + + fn load_with_short_home( + candidate: &Candidate, + allow_missing: bool, + ) -> Result { let root = candidate.state_root.join("run/smolvm"); reject_aliased_state(&root)?; + if allow_missing { + match fs::symlink_metadata(root.join("owner.pending")) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + _ => { + return Err(CandidateError::new( + "recovery_required", + "Provider owner update is pending or unobservable; HOME alias was not restored.", + )); + } + } + } let owner: Self = read(&root.join("owner.json"))?; owner.network.validate()?; if let Some(share) = &owner.project_share { @@ -250,20 +275,69 @@ impl Owner { "Provider owner identity does not match this checkout.", )); } - // The only permitted alias is an exact, receipt-bound short HOME for Unix sockets. - let alias = fs::symlink_metadata(&owner.short_home).map_err(io)?; + owner.check_short_home(candidate, allow_missing)?; + check_private_directory(&root.join("home"))?; + Ok(owner) + } + /// The only permitted alias is the exact receipt-bound HOME for Unix sockets. + fn check_short_home( + &self, + candidate: &Candidate, + allow_missing: bool, + ) -> Result<(), CandidateError> { + let alias = match fs::symlink_metadata(&self.short_home) { + Ok(alias) => alias, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return if allow_missing { + Ok(()) + } else { + Err(CandidateError::new( + "provider_home_missing", + "Temporary provider HOME alias is absent. Use runtime recover to verify the stopped pool before restoring it.", + )) + }; + } + Err(error) => return Err(io(error)), + }; if !alias.file_type().is_symlink() || alias.uid() != unsafe { libc::geteuid() } - || fs::read_link(&owner.short_home).map_err(io)? != root.join("home") + || fs::read_link(&self.short_home).map_err(io)? + != candidate.state_root.join("run/smolvm/home") { return Err(CandidateError::new( "foreign_state", "Short provider HOME alias was replaced.", )); } - check_private_directory(&root.join("home"))?; - Ok(owner) + Ok(()) } + + /// Called only while the lifecycle owns the pool and VM locks and has audited absence. + /// Recheck the durable selection, then create exclusively; a concurrent alias is retained. + pub(super) fn restore_missing_short_home( + &self, + candidate: &Candidate, + ) -> Result<(), CandidateError> { + let observed = Self::load_for_short_home_recovery(candidate)?; + if observed != *self { + return Err(CandidateError::new( + "foreign_state", + "Provider owner changed during HOME recovery; no alias was restored.", + )); + } + std::os::unix::fs::symlink( + candidate.state_root.join("run/smolvm/home"), + &self.short_home, + ) + .map_err(|_| { + CandidateError::new( + "socket_alias_collision", + "Cannot exclusively restore provider HOME alias; no existing path was replaced.", + ) + })?; + self.check_short_home(candidate, false) + } + #[cfg(all(test, target_os = "macos"))] pub fn create( candidate: &Candidate, From 8f99174def8cd7cdd5e0d0187582e5570564f892 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Tue, 29 Sep 2026 22:12:49 -0400 Subject: [PATCH 2/9] fix: explicitly recover legacy host filesystem device identities --- docs/guides/native-candidate.md | 32 ++ packages/runtime-core/src/main.rs | 28 ++ .../runtime-core/src/provider/lifecycle.rs | 1 + .../src/provider/lifecycle/host_filesystem.rs | 267 ++++++++++++++++ .../lifecycle/host_filesystem/tests.rs | 294 ++++++++++++++++++ packages/runtime-core/src/provider/mod.rs | 1 + packages/runtime-core/src/provider/state.rs | 2 +- 7 files changed, 624 insertions(+), 1 deletion(-) create mode 100644 packages/runtime-core/src/provider/lifecycle/host_filesystem.rs create mode 100644 packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 2dbe382d3..752281131 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -133,6 +133,38 @@ data, and requires inspection before retry. No guest is started by alias recover Interrupted starts without a recorded process or both identified disks remain outside this repair path. +A physical macOS reboot may also renumber the mounted filesystem device. Strict +disk and source checks still refuse a changed device number; missing-HOME recovery +does not waive them. For an offline **stock pool**, inspect the separate migration: + +```sh +./hack-native --candidate-root /absolute/private/candidate-home runtime host-filesystem-recovery --json +./hack-native --candidate-root /absolute/private/candidate-home runtime recover-host-filesystem --expect-sha256 --accept-legacy-device-rebind --json +./hack-native --candidate-root /absolute/private/candidate-home runtime recover --json +``` + +Review the inspection before selecting its hash. This explicit legacy migration +requires an absent recorded provider whose start predates the current host boot, +no active provider commands, exclusive existing operation/VM locks and closed disk +handles. Both disk inodes, sizes and ext4 UUIDs, the exact source path/inode and its +ownership must be unchanged; only one common old-to-new device-number change is +allowed. The inspection is read-only. Publication atomically changes only the +owner's disk and source device numbers, retaining its phase and process record. +Normal commands keep their strict identity checks. + +Legacy receipts have no original host boot UUID or filesystem volume UUID. Calendar +timestamps corroborate a reboot, and matching retained file identities constrain +the migration, but neither proves original volume continuity. The opt-in explicitly +accepts that limitation; copied or relocated pools are outside this procedure. +Prepared-base pools and pending owner/network/activation updates require separate +recovery and are refused. A torn owner publication preserves `owner.pending` and +blocks another migration; do not delete or adopt that file manually. + +This does not start the VM, restore HOME, retire stale sockets or rewrite historical +graph receipts. Use ordinary recovery afterward. Historical shared-source graphs +retain the prior identity and may require verified cleanup plus a new generation; +successful metadata migration alone does not establish application recovery. + Compatibility is qualified for specific bundle hashes and state formats. The displayed version alone does not establish frontend/executor or downgrade compatibility. V4 and candidate homes remain separate; this procedure does not diff --git a/packages/runtime-core/src/main.rs b/packages/runtime-core/src/main.rs index 68bdc05ed..be6ee57a7 100644 --- a/packages/runtime-core/src/main.rs +++ b/packages/runtime-core/src/main.rs @@ -86,6 +86,8 @@ Usage: hack-local runtime up --profile development --internet --json hack-local runtime network extend --allow-host [--allow-host ] --json hack-local runtime up|status|down|recover [--json] + hack-local runtime host-filesystem-recovery [--json] + hack-local runtime recover-host-filesystem --expect-sha256 --accept-legacy-device-rebind [--json] hack-local node serve|status|inspect hack-local node request hack-local --version @@ -857,6 +859,32 @@ fn run() -> Result<(), CandidateError> { image, )?)?; } + ["runtime", "host-filesystem-recovery"] + | ["runtime", "host-filesystem-recovery", "--json"] => { + print_json(&hack_runtime_core::provider::host_filesystem::inspect( + &discover_candidate(&requested)?, + )?)?; + } + [ + "runtime", + "recover-host-filesystem", + "--expect-sha256", + hash, + "--accept-legacy-device-rebind", + ] + | [ + "runtime", + "recover-host-filesystem", + "--expect-sha256", + hash, + "--accept-legacy-device-rebind", + "--json", + ] => { + print_json(&hack_runtime_core::provider::host_filesystem::recover( + &discover_candidate(&requested)?, + hash, + )?)?; + } ["runtime", action] | ["runtime", action, "--json"] => { let candidate = discover_candidate(&requested)?; let result = match *action { diff --git a/packages/runtime-core/src/provider/lifecycle.rs b/packages/runtime-core/src/provider/lifecycle.rs index 8b47b97b0..36decce95 100644 --- a/packages/runtime-core/src/provider/lifecycle.rs +++ b/packages/runtime-core/src/provider/lifecycle.rs @@ -1,3 +1,4 @@ +pub mod host_filesystem; mod interrupted; mod prepared_boot; #[cfg(any(target_os = "macos", test))] diff --git a/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs b/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs new file mode 100644 index 000000000..08467b8f3 --- /dev/null +++ b/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs @@ -0,0 +1,267 @@ +//! Explicit legacy device-number migration for an offline stock pool. +//! +//! Old receipts have neither a host boot UUID nor a filesystem volume UUID. Calendar start +//! time, unchanged inode/size/ext4 UUID and exact paths constrain this migration, but cannot +//! establish original volume continuity. Only an explicitly accepted, hash-selected inspection +//! may update the owner. Normal runtime operations retain their strict identity comparisons. +use super::{Owner, binary, identity, lock_absent_disks, root, state}; +use crate::{Candidate, CandidateError}; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::{fs, os::unix::fs::MetadataExt}; + +#[derive(Debug, Serialize, PartialEq, Eq)] +pub struct Inspection { + pub schema: &'static str, + pub selection_sha256: String, + pub owner_sha256: String, + pub machine: String, + pub host_boot_micros: u64, + pub provider_start_micros: u64, + pub old_device: u64, + pub new_device: u64, + pub pool_inode: u64, + pub storage: identity::DiskIdentity, + pub overlay: identity::DiskIdentity, + pub project_share: Option, + pub qualification: &'static str, +} + +fn refused(detail: &str) -> CandidateError { + CandidateError::new( + "host_filesystem_recovery", + format!("{detail}; no identity was changed."), + ) +} + +#[cfg(target_os = "macos")] +fn host_boot_micros() -> Result { + let mut time = std::mem::MaybeUninit::::zeroed(); + let mut length = std::mem::size_of::(); + // SAFETY: read-only sysctl writes an exactly sized timeval; no input or retained pointers. + if unsafe { + libc::sysctlbyname( + c"kern.boottime".as_ptr(), + time.as_mut_ptr().cast(), + &mut length, + std::ptr::null_mut(), + 0, + ) + } != 0 + || length != std::mem::size_of::() + { + return Err(refused("Native host boot time is unavailable")); + } + // SAFETY: a complete timeval was returned above. + let time = unsafe { time.assume_init() }; + let seconds = u64::try_from(time.tv_sec).map_err(|_| refused("Invalid host boot time"))?; + let micros = u64::try_from(time.tv_usec).map_err(|_| refused("Invalid host boot time"))?; + if micros >= 1_000_000 { + return Err(refused("Invalid host boot time")); + } + seconds + .checked_mul(1_000_000) + .and_then(|v| v.checked_add(micros)) + .filter(|v| *v > 0) + .ok_or_else(|| refused("Invalid host boot time")) +} + +#[cfg(not(target_os = "macos"))] +fn host_boot_micros() -> Result { + Err(CandidateError::new( + "unsupported_host", + "Legacy filesystem recovery requires macOS.", + )) +} + +fn no_auxiliary_update(candidate: &Candidate) -> Result<(), CandidateError> { + crate::provider::network_update::require_complete(candidate)?; + for name in [ + "owner.pending", + "prepared-base.json", + "prepared-base.json.pending", + ] { + match fs::symlink_metadata(root(candidate).join(name)) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + _ => { + return Err(refused( + "Pending owner or prepared-base state requires separate recovery", + )); + } + } + } + Ok(()) +} + +fn dead_provider(candidate: &Candidate, owner: &Owner, boot: u64) -> Result<(), CandidateError> { + let process = owner + .process + .as_ref() + .ok_or_else(|| refused("No recorded provider identity"))?; + // SAFETY: geteuid has no preconditions. + identity::verify(process, process, &binary(candidate), unsafe { + libc::geteuid() + })?; + if !owner.created + || process.start_micros >= boot + || identity::alive(process.pid)? + || identity::executable_running(&binary(candidate))? + { + return Err(refused( + "Provider must be absent and its recorded start must predate this host boot", + )); + } + Ok(()) +} + +fn same_disk(before: &identity::DiskIdentity, after: &identity::DiskIdentity) -> bool { + before.device != after.device + && before.inode == after.inode + && before.bytes == after.bytes + && before.uuid == after.uuid +} + +/// Build a candidate owner in memory. This function neither adopts a new disk nor writes state. +fn selection( + candidate: &Candidate, + owner: &Owner, + boot: u64, +) -> Result<(Inspection, Owner), CandidateError> { + no_auxiliary_update(candidate)?; + dead_provider(candidate, owner, boot)?; + let data = owner.real_data_dir(candidate)?; + let storage = identity::disk(&data.join("storage.raw"))?; + let overlay = identity::disk(&data.join("overlay.raw"))?; + let before = owner + .storage + .as_ref() + .ok_or_else(|| refused("Storage identity is missing"))?; + let prior_overlay = owner + .overlay + .as_ref() + .ok_or_else(|| refused("Overlay identity is missing"))?; + let metadata = fs::symlink_metadata(root(candidate)).map_err(state::io)?; + if !same_disk(before, &storage) + || !same_disk(prior_overlay, &overlay) + || before.device != prior_overlay.device + || storage.device != overlay.device + || storage.device != metadata.dev() + { + return Err(refused( + "Only a common device-number change with unchanged disk inode, size and UUID is accepted", + )); + } + let mut next = owner.clone(); + if let Some(share) = &owner.project_share { + let observed = + crate::provider::ProjectShareIntent::approve(&share.project, share.unfiltered_source)?; + let mut expected = share.clone(); + expected.device = storage.device; + if share.device != before.device || observed != expected { + return Err(refused( + "Project share changed beyond the same filesystem device number", + )); + } + next.project_share = Some(observed); + } + next.storage = Some(storage.clone()); + next.overlay = Some(overlay.clone()); + // The canonical read is bounded and validated independently of raw-byte hashing. + let bytes = crate::provider::prepared_base::read_private( + &root(candidate).join("owner.json"), + 1024 * 1024, + )?; + if Owner::load_for_short_home_recovery(candidate)? != *owner { + return Err(refused("Owner selection changed")); + } + let mut inspection = Inspection { + schema: "hack.host-filesystem-recovery/v1", + selection_sha256: String::new(), + owner_sha256: format!("{:x}", Sha256::digest(bytes)), + machine: owner.machine.clone(), + host_boot_micros: boot, + provider_start_micros: owner + .process + .as_ref() + .ok_or_else(|| refused("No process"))? + .start_micros, + old_device: before.device, + new_device: storage.device, + pool_inode: metadata.ino(), + storage, + overlay, + project_share: next.project_share.clone(), + qualification: "explicit-legacy-migration-original-volume-continuity-unproven", + }; + inspection.selection_sha256 = format!( + "{:x}", + Sha256::digest( + serde_json::to_vec(&inspection).map_err(|_| refused("Cannot encode selection"))? + ) + ); + Ok((inspection, next)) +} + +/// Read-only inspection holds both existing locks and proves disks have no open handles. +pub fn inspect(candidate: &Candidate) -> Result { + let _operation = state::Lock::acquire_existing(&root(candidate))?; + let owner = Owner::load_for_short_home_recovery(candidate)?; + let (inspection, next) = selection(candidate, &owner, host_boot_micros()?)?; + let _vm = lock_absent_disks(candidate, &next)?; + if selection(candidate, &owner, host_boot_micros()?)?.0 != inspection { + return Err(refused("Inspection changed during absence verification")); + } + Ok(inspection) +} + +/// Publish disk and source device changes in ONE atomic owner replacement. A crash leaves +/// old or new committed metadata; any unfinished owner.pending is preserved and blocks retry. +/// This does not launch, restore the alias, retire historical sockets, or rewrite graph receipts. +pub fn recover(candidate: &Candidate, expected: &str) -> Result { + recover_with_boot(candidate, expected, host_boot_micros) +} + +fn recover_with_boot( + candidate: &Candidate, + expected: &str, + boot: impl Fn() -> Result, +) -> Result { + if expected.len() != 64 || !expected.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(refused("An exact inspection SHA-256 is required")); + } + let _operation = state::Lock::acquire_existing(&root(candidate))?; + let owner = Owner::load_for_short_home_recovery(candidate)?; + let (inspection, next) = selection(candidate, &owner, boot()?)?; + if inspection.selection_sha256 != expected { + return Err(refused("Inspection selection is stale")); + } + let _vm = lock_absent_disks(candidate, &next)?; + if selection(candidate, &owner, boot()?)?.0 != inspection { + return Err(refused("Selected identities changed before publication")); + } + next.save(candidate)?; + if Owner::load_for_short_home_recovery(candidate)? != next { + return Err(CandidateError::new( + "host_filesystem_recovery_incomplete", + "Owner was published but changed during confirmation; retained state requires inspection.", + )); + } + super::verify_disks(candidate, &next).map_err(|e| { + CandidateError::new( + "host_filesystem_recovery_incomplete", + format!( + "Owner was published but disk confirmation failed ({}); inspect retained state.", + e.code + ), + ) + })?; + if let Some(share) = &next.project_share { + share.validate().map_err(|e| CandidateError::new( + "host_filesystem_recovery_incomplete", format!("Owner was published but source confirmation failed ({}); inspect retained state.", e.code) + ))?; + } + Ok(inspection) +} + +#[cfg(all(test, target_os = "macos"))] +mod tests; diff --git a/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs b/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs new file mode 100644 index 000000000..3585833bf --- /dev/null +++ b/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs @@ -0,0 +1,294 @@ +use super::*; +use crate::provider::{NetworkIntent, Profile, ProjectShareIntent}; +use std::{fs::File, os::fd::AsRawFd, path::PathBuf, process::Command}; + +struct Pool { + candidate: Candidate, + owner: Owner, + directory: PathBuf, + boot: u64, +} +impl Pool { + fn new() -> Self { + let mut child = Command::new("/bin/sleep").arg("30").spawn().unwrap(); + let mut process = identity::observe(child.id() as i32).unwrap(); + child.kill().unwrap(); + child.wait().unwrap(); + let boot = process.start_micros + 1; + let directory = fs::canonicalize(std::env::temp_dir()) + .unwrap() + .join(format!( + "hack-device-recovery-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + state::private_directory(&directory).unwrap(); + let candidate = Candidate::discover(&directory).unwrap(); + let operation = state::Lock::acquire(&root(&candidate)).unwrap(); + let project = directory.join("app"); + state::private_directory(&project).unwrap(); + fs::write(project.join("package.json"), b"{}").unwrap(); + let share = ProjectShareIntent::approve(&project, true).unwrap(); + let mut owner = Owner::create_with_project_share( + &candidate, + Profile::Development, + None, + NetworkIntent::Isolated, + None, + Some(share), + ) + .unwrap(); + let data = owner.real_data_dir(&candidate).unwrap(); + state::private_directory(&data).unwrap(); + for (name, tag) in [("storage.raw", 1u8), ("overlay.raw", 2)] { + let mut bytes = vec![0; 4096]; + bytes[1080..1082].copy_from_slice(&[0x53, 0xef]); + bytes[1128..1144].fill(tag); + bytes[2048..2059].copy_from_slice(b"data-marker"); + fs::write(data.join(name), bytes).unwrap(); + } + fs::write(data.join("name"), &owner.machine).unwrap(); + fs::write(data.join("vm.lock"), b"").unwrap(); + process.executable = binary(&candidate); + owner.process = Some(process); + owner.created = true; + owner.phase = "running".into(); + let mut storage = identity::disk(&data.join("storage.raw")).unwrap(); + let mut overlay = identity::disk(&data.join("overlay.raw")).unwrap(); + let old = storage.device + 1; + storage.device = old; + overlay.device = old; + owner.storage = Some(storage); + owner.overlay = Some(overlay); + owner.project_share.as_mut().unwrap().device = old; + owner.save(&candidate).unwrap(); + fs::remove_file(&owner.short_home).unwrap(); + drop(operation); + Self { + candidate, + owner, + directory, + boot, + } + } + fn data(&self) -> PathBuf { + self.owner.real_data_dir(&self.candidate).unwrap() + } + fn selected(&self) -> Inspection { + selection(&self.candidate, &self.owner, self.boot) + .unwrap() + .0 + } + fn receipt(&self) -> Vec { + fs::read(root(&self.candidate).join("owner.json")).unwrap() + } + fn apply(&self, hash: &str) -> Result { + recover_with_boot(&self.candidate, hash, || Ok(self.boot)) + } + fn unchanged(&self, before: &[u8]) { + assert_eq!(self.receipt(), before); + assert!(self.owner.short_home.symlink_metadata().is_err()); + } +} +impl Drop for Pool { + fn drop(&mut self) { + let _ = fs::remove_file(&self.owner.short_home); + fs::remove_dir_all(&self.directory).unwrap(); + } +} + +#[test] +fn migration_changes_only_devices_and_keeps_alias_graph_history_and_data() { + let pool = Pool::new(); + let old = pool.receipt(); + let storage = fs::read(pool.data().join("storage.raw")).unwrap(); + let overlay = fs::read(pool.data().join("overlay.raw")).unwrap(); + let history = pool.directory.join("historical-graph.json"); + fs::write(&history, b"historical-source-binding").unwrap(); + let inspected = pool.selected(); + pool.unchanged(&old); + let result = pool.apply(&inspected.selection_sha256).unwrap(); + assert_eq!(result, inspected); + let mut expected = pool.owner.clone(); + expected.storage.as_mut().unwrap().device = inspected.new_device; + expected.overlay.as_mut().unwrap().device = inspected.new_device; + expected.project_share.as_mut().unwrap().device = inspected.new_device; + assert_eq!( + Owner::load_for_short_home_recovery(&pool.candidate).unwrap(), + expected + ); + assert!(pool.owner.short_home.symlink_metadata().is_err()); + assert_eq!(fs::read(history).unwrap(), b"historical-source-binding"); + assert_eq!(fs::read(pool.data().join("storage.raw")).unwrap(), storage); + assert_eq!(fs::read(pool.data().join("overlay.raw")).unwrap(), overlay); + assert!(pool.apply(&inspected.selection_sha256).is_err()); + // Ordinary recovery still owns HOME restoration and the recovered phase. + let recovered = crate::provider::recover(&pool.candidate).unwrap(); + assert_eq!(recovered.phase, "recovered-unclean"); + assert_eq!(recovered.process_alive, Some(false)); + assert_eq!(fs::read(pool.data().join("storage.raw")).unwrap(), storage); +} + +#[test] +fn disk_and_share_changes_beyond_common_device_refuse_without_writes() { + for case in 0..7 { + let mut pool = Pool::new(); + let observed_device = identity::disk(&pool.data().join("storage.raw")) + .unwrap() + .device; + let disk = pool.owner.storage.as_mut().unwrap(); + match case { + 0 => disk.inode += 1, + 1 => disk.bytes += 1, + 2 => disk.uuid.push('0'), + 3 => disk.device = observed_device, + 4 => pool.owner.overlay.as_mut().unwrap().device += 1, + 5 => pool.owner.project_share.as_mut().unwrap().inode += 1, + _ => pool.owner.project_share.as_mut().unwrap().device += 1, + } + pool.owner.save(&pool.candidate).unwrap(); + let before = pool.receipt(); + assert!(pool.apply(&"a".repeat(64)).is_err()); + pool.unchanged(&before); + } +} + +#[test] +fn stale_owner_or_host_boot_selection_refuses() { + let mut pool = Pool::new(); + let selection = pool.selected(); + assert!(pool.apply(&"a".repeat(64)).is_err()); + pool.owner.phase = "stopped".into(); + pool.owner.save(&pool.candidate).unwrap(); + let before = pool.receipt(); + assert!(pool.apply(&selection.selection_sha256).is_err()); + pool.unchanged(&before); + let selected = pool.selected(); + assert!( + recover_with_boot(&pool.candidate, &selected.selection_sha256, || Ok(pool + .boot + + 1)) + .is_err() + ); + pool.unchanged(&before); +} + +#[test] +fn same_boot_live_reused_pid_and_unknown_identity_refuse() { + for case in 0..4 { + let mut pool = Pool::new(); + if case == 0 { + pool.boot = pool.owner.process.as_ref().unwrap().start_micros; + } + if case == 1 || case == 2 { + let p = pool.owner.process.as_mut().unwrap(); + p.pid = std::process::id() as i32; + if case == 1 { + p.start_micros = identity::observe(p.pid).unwrap().start_micros; + } + } + if case == 3 { + pool.owner.process = None; + } + pool.owner.save(&pool.candidate).unwrap(); + let before = pool.receipt(); + assert!(pool.apply(&"a".repeat(64)).is_err()); + pool.unchanged(&before); + } +} + +#[test] +fn held_operation_vm_lock_and_disk_handles_refuse() { + for case in 0..3 { + let pool = Pool::new(); + let selected = pool.selected(); + let before = pool.receipt(); + let _operation = + (case == 0).then(|| state::Lock::acquire_existing(&root(&pool.candidate)).unwrap()); + let file = if case == 1 { + Some(File::open(pool.data().join("vm.lock")).unwrap()) + } else if case == 2 { + Some(File::open(pool.data().join("storage.raw")).unwrap()) + } else { + None + }; + if case == 1 { + assert_eq!( + unsafe { + libc::flock( + file.as_ref().unwrap().as_raw_fd(), + libc::LOCK_EX | libc::LOCK_NB, + ) + }, + 0 + ); + } + assert!(pool.apply(&selected.selection_sha256).is_err()); + pool.unchanged(&before); + } +} + +#[test] +fn pending_foreign_updates_and_prepared_pools_are_preserved() { + for name in [ + "owner.pending", + "network-update.json", + "network-update.pending", + "prepared-base.json", + "prepared-base.json.pending", + ] { + let pool = Pool::new(); + let selected = pool.selected(); + let before = pool.receipt(); + let path = root(&pool.candidate).join(name); + fs::write(&path, b"foreign-or-interrupted").unwrap(); + assert!(pool.apply(&selected.selection_sha256).is_err()); + pool.unchanged(&before); + assert_eq!(fs::read(path).unwrap(), b"foreign-or-interrupted"); + } +} + +#[test] +fn foreign_alias_and_source_substitution_refuse() { + for case in 0..2 { + let pool = Pool::new(); + let selected = pool.selected(); + let before = pool.receipt(); + if case == 0 { + fs::write(&pool.owner.short_home, b"foreign").unwrap(); + } else { + let project = &pool.owner.project_share.as_ref().unwrap().project; + fs::rename(project, pool.directory.join("preserved-app")).unwrap(); + state::private_directory(project).unwrap(); + fs::write(project.join("package.json"), b"{}").unwrap(); + } + assert!(pool.apply(&selected.selection_sha256).is_err()); + assert_eq!(pool.receipt(), before); + if case == 0 { + assert_eq!(fs::read(&pool.owner.short_home).unwrap(), b"foreign"); + } + } +} + +#[test] +fn final_recheck_rejects_identity_change_before_publication() { + let pool = Pool::new(); + let selected = pool.selected(); + let before = pool.receipt(); + let called = std::cell::Cell::new(false); + let result = recover_with_boot(&pool.candidate, &selected.selection_sha256, || { + if called.replace(true) { + fs::write(pool.data().join("storage.raw"), b"changed").unwrap(); + } + Ok(pool.boot) + }); + assert!(result.is_err()); + pool.unchanged(&before); + assert_eq!( + fs::read(pool.data().join("storage.raw")).unwrap(), + b"changed" + ); +} diff --git a/packages/runtime-core/src/provider/mod.rs b/packages/runtime-core/src/provider/mod.rs index 30f7082bf..e851f1890 100644 --- a/packages/runtime-core/src/provider/mod.rs +++ b/packages/runtime-core/src/provider/mod.rs @@ -45,6 +45,7 @@ pub mod resources; pub mod storage_usage; pub use image_load::load as load_image; mod lifecycle; +pub use lifecycle::host_filesystem; mod network_intent; mod network_update; pub use network_update::{enable_internet, extend_network}; diff --git a/packages/runtime-core/src/provider/state.rs b/packages/runtime-core/src/provider/state.rs index e8810dbe1..bc4c3bf84 100644 --- a/packages/runtime-core/src/provider/state.rs +++ b/packages/runtime-core/src/provider/state.rs @@ -176,7 +176,7 @@ impl Default for ReclamationPolicy { } } -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Owner { #[serde(default, skip_serializing_if = "Option::is_none")] From 4b5bdff9ac6cc97d3d13970aac6ce5a6c59b0435 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Tue, 29 Sep 2026 22:16:52 -0400 Subject: [PATCH 3/9] fix: fence replaced lock paths before filesystem recovery --- .../src/provider/lifecycle/host_filesystem.rs | 40 ++++++++++++++++--- .../lifecycle/host_filesystem/tests.rs | 25 ++++++++++++ 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs b/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs index 08467b8f3..6c9e10634 100644 --- a/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs +++ b/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs @@ -8,7 +8,7 @@ use super::{Owner, binary, identity, lock_absent_disks, root, state}; use crate::{Candidate, CandidateError}; use serde::Serialize; use sha2::{Digest, Sha256}; -use std::{fs, os::unix::fs::MetadataExt}; +use std::{fs, os::unix::fs::MetadataExt, path::Path}; #[derive(Debug, Serialize, PartialEq, Eq)] pub struct Inspection { @@ -121,6 +121,34 @@ fn same_disk(before: &identity::DiskIdentity, after: &identity::DiskIdentity) -> && before.uuid == after.uuid } +/// A retained flock cannot fence a writer that opens a substituted lock pathname. +fn bound_locks( + candidate: &Candidate, + owner: &Owner, + operation: &state::Lock, + vm: &fs::File, +) -> Result<(), CandidateError> { + let check = |path: &Path, expected: (u64, u64)| -> Result<(), CandidateError> { + let metadata = fs::symlink_metadata(path).map_err(state::io)?; + if !metadata.is_file() + || metadata.nlink() != 1 + || (metadata.dev(), metadata.ino()) != expected + { + return Err(refused("Held lock pathname was replaced")); + } + Ok(()) + }; + check( + &root(candidate).join("operation.lock"), + operation.identity()?, + )?; + let metadata = vm.metadata().map_err(state::io)?; + check( + &owner.real_data_dir(candidate)?.join("vm.lock"), + (metadata.dev(), metadata.ino()), + ) +} + /// Build a candidate owner in memory. This function neither adopts a new disk nor writes state. fn selection( candidate: &Candidate, @@ -204,13 +232,14 @@ fn selection( /// Read-only inspection holds both existing locks and proves disks have no open handles. pub fn inspect(candidate: &Candidate) -> Result { - let _operation = state::Lock::acquire_existing(&root(candidate))?; + let operation = state::Lock::acquire_existing(&root(candidate))?; let owner = Owner::load_for_short_home_recovery(candidate)?; let (inspection, next) = selection(candidate, &owner, host_boot_micros()?)?; - let _vm = lock_absent_disks(candidate, &next)?; + let vm = lock_absent_disks(candidate, &next)?; if selection(candidate, &owner, host_boot_micros()?)?.0 != inspection { return Err(refused("Inspection changed during absence verification")); } + bound_locks(candidate, &owner, &operation, &vm)?; Ok(inspection) } @@ -229,16 +258,17 @@ fn recover_with_boot( if expected.len() != 64 || !expected.bytes().all(|b| b.is_ascii_hexdigit()) { return Err(refused("An exact inspection SHA-256 is required")); } - let _operation = state::Lock::acquire_existing(&root(candidate))?; + let operation = state::Lock::acquire_existing(&root(candidate))?; let owner = Owner::load_for_short_home_recovery(candidate)?; let (inspection, next) = selection(candidate, &owner, boot()?)?; if inspection.selection_sha256 != expected { return Err(refused("Inspection selection is stale")); } - let _vm = lock_absent_disks(candidate, &next)?; + let vm = lock_absent_disks(candidate, &next)?; if selection(candidate, &owner, boot()?)?.0 != inspection { return Err(refused("Selected identities changed before publication")); } + bound_locks(candidate, &owner, &operation, &vm)?; next.save(candidate)?; if Owner::load_for_short_home_recovery(candidate)? != next { return Err(CandidateError::new( diff --git a/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs b/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs index 3585833bf..4c6cc502f 100644 --- a/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs +++ b/packages/runtime-core/src/provider/lifecycle/host_filesystem/tests.rs @@ -292,3 +292,28 @@ fn final_recheck_rejects_identity_change_before_publication() { b"changed" ); } + +#[test] +fn substituted_lock_paths_do_not_authorize_publication_on_an_old_descriptor() { + for name in ["operation.lock", "vm.lock"] { + let pool = Pool::new(); + let selected = pool.selected(); + let before = pool.receipt(); + let called = std::cell::Cell::new(false); + let path = if name == "operation.lock" { + root(&pool.candidate).join(name) + } else { + pool.data().join(name) + }; + let result = recover_with_boot(&pool.candidate, &selected.selection_sha256, || { + if called.replace(true) { + fs::rename(&path, path.with_extension("preserved")).unwrap(); + fs::write(&path, b"replacement lock").unwrap(); + } + Ok(pool.boot) + }); + assert!(matches!(result, Err(e) if e.code == "host_filesystem_recovery")); + pool.unchanged(&before); + assert_eq!(fs::read(path).unwrap(), b"replacement lock"); + } +} From 3266931063bf33339966bcff4f6ee7d497387631 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 01:45:05 -0400 Subject: [PATCH 4/9] fix: explicitly recover retained graph host pins after reboot --- docs/guides/native-candidate.md | 35 + packages/runtime-core/src/graph_cli.rs | 44 ++ packages/runtime-core/src/main.rs | 2 + .../src/provider/graph/bridges/cleanup.rs | 78 +- .../src/provider/graph/dead_owner_cleanup.rs | 194 ++++- .../src/provider/graph/dependency_slots.rs | 165 +++- .../src/provider/graph/foreground.rs | 1 - .../provider/graph/foreground/native_test.rs | 1 + .../native_test/host_pin_recovery.rs | 537 +++++++++++++ .../provider/graph/foreground/transport.rs | 202 ++++- .../graph/foreground/transport_tests.rs | 72 ++ .../src/provider/graph/host_pin_recovery.rs | 704 ++++++++++++++++++ .../src/provider/graph/host_relay.rs | 23 +- .../src/provider/graph/live_owner_cleanup.rs | 2 +- .../runtime-core/src/provider/graph/mod.rs | 4 + .../runtime-core/src/provider/host_pin.rs | 40 + .../src/provider/lifecycle/host_filesystem.rs | 4 +- packages/runtime-core/src/provider/mod.rs | 2 + .../src/provider/relay_owner/publication.rs | 62 +- .../provider/relay_owner/publication/dead.rs | 1 + 20 files changed, 2099 insertions(+), 74 deletions(-) create mode 100644 packages/runtime-core/src/provider/graph/foreground/native_test/host_pin_recovery.rs create mode 100644 packages/runtime-core/src/provider/graph/host_pin_recovery.rs create mode 100644 packages/runtime-core/src/provider/host_pin.rs diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 752281131..010e75fff 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -165,6 +165,41 @@ graph receipts. Use ordinary recovery afterward. Historical shared-source graphs retain the prior identity and may require verified cleanup plus a new generation; successful metadata migration alone does not establish application recovery. +After that explicit provider recovery and one audited VM boot, a retained graph +from the immediately preceding guest boot can still carry old host device numbers +in its publisher, relay-control, and dependency-socket receipts. Inspect and select +one run's host-pin recovery separately: + +```sh +./hack-native --candidate-root /absolute/private/candidate-home graph inspect-host-pin-recovery --run-id --json +./hack-native --candidate-root /absolute/private/candidate-home graph recover-host-pins --run-id --expect-selection --accept-legacy-device-rebind --json +./hack-native --candidate-root /absolute/private/candidate-home graph recover-cleanup --run-id --expect-receipt --json +./hack-native --candidate-root /absolute/private/candidate-home graph retire-recovered-publisher --run-id --expect-owner --json +``` + +The first command only inspects. The second publishes a private, exact-run +witness before cleanup; it changes no graph, publisher, control, or dependency +receipt. It requires the provider's repaired current disks/source, unchanged +recorded inodes and raw receipts, dead owners whose recorded starts precede the +current physical host boot, refused socket listeners, and the immediate guest +boot transition. It also binds retained volume names and actual labels. Cleanup +and retirement can use the witness only for those selected old pins; ordinary +reads and later publisher generations remain strict. Missing or foreign pins, +active listeners, changed resources, pending journals, or further guest boots +refuse without adopting another run. A completed older cleanup journal is +retained as history and does not itself block a later selected generation. + +Legacy receipts do not identify the original APFS volume. This explicit +device-number rebind cannot prove pre-reboot volume continuity. Completing these +commands retains the old run's data and proves cleanup of its dead generation; +it does not migrate `graph.source.shared`, restore the application, establish +route readiness, or claim overall v5 acceptance. Same-run source continuity +requires a separate explicit witness-bound transition after cleanup and +publisher retirement. If macOS removed the foreground or relay-control +directory itself during reboot, this command refuses: the old pin receipts no +longer exist, and absent pathnames cannot stand in for their recorded owner +identities. That case requires a separate selected absence-recovery procedure. + Compatibility is qualified for specific bundle hashes and state formats. The displayed version alone does not establish frontend/executor or downgrade compatibility. V4 and candidate homes remain separate; this procedure does not diff --git a/packages/runtime-core/src/graph_cli.rs b/packages/runtime-core/src/graph_cli.rs index 9b9e3b52c..b93a6872a 100644 --- a/packages/runtime-core/src/graph_cli.rs +++ b/packages/runtime-core/src/graph_cli.rs @@ -25,6 +25,50 @@ pub fn command(candidate: &Candidate, args: &[&str]) -> Result run, + _ => return Err(invalid()), + }; + #[cfg(target_os = "macos")] + { + return graph::inspect_host_pin_recovery(candidate, run); + } + #[cfg(not(target_os = "macos"))] + { + let _ = run; + return Err(invalid()); + } + } + if *action == "recover-host-pins" { + let (run, expected) = match *args { + [ + "--run-id", + run, + "--expect-selection", + expected, + "--accept-legacy-device-rebind", + ] + | [ + "--run-id", + run, + "--expect-selection", + expected, + "--accept-legacy-device-rebind", + "--json", + ] => (run, expected), + _ => return Err(invalid()), + }; + #[cfg(target_os = "macos")] + { + return graph::recover_host_pins(candidate, run, expected); + } + #[cfg(not(target_os = "macos"))] + { + let _ = (run, expected); + return Err(invalid()); + } + } if ["run-selection", "run-service"].contains(action) { return one_off::command(candidate, action, args); } diff --git a/packages/runtime-core/src/main.rs b/packages/runtime-core/src/main.rs index be6ee57a7..5878fae07 100644 --- a/packages/runtime-core/src/main.rs +++ b/packages/runtime-core/src/main.rs @@ -27,6 +27,8 @@ Usage: hack-local graph recover-cleanup --run-id <32-hex> --expect-receipt [--json] hack-local graph recover-live-owner --run-id <32-hex> --expect-receipt [--json] hack-local graph retire-recovered-publisher --run-id <32-hex> --expect-owner <32-hex> [--json] + hack-local graph inspect-host-pin-recovery --run-id <32-hex> [--json] + hack-local graph recover-host-pins --run-id <32-hex> --expect-selection <64-hex> --accept-legacy-device-rebind [--json] hack-local graph logs --run-id <32-hex> --service [--tail <1..1000>] [--json] hack-local graph exec --run-id <32-hex> --service [--workdir /path] [--timeout-seconds <1..120>] [--json] -- [args...] hack-local graph dependency-plan --dependencies [--json] diff --git a/packages/runtime-core/src/provider/graph/bridges/cleanup.rs b/packages/runtime-core/src/provider/graph/bridges/cleanup.rs index 8d918fe5f..c76005ce1 100644 --- a/packages/runtime-core/src/provider/graph/bridges/cleanup.rs +++ b/packages/runtime-core/src/provider/graph/bridges/cleanup.rs @@ -46,6 +46,14 @@ fn validate_selection( engine: &Engine<'_>, receipt: &Receipt, selection: &Selection, +) -> Result<(), CandidateError> { + validate_selection_recovery(engine, receipt, selection, None) +} +fn validate_selection_recovery( + engine: &Engine<'_>, + receipt: &Receipt, + selection: &Selection, + host_pin: Option<&super::super::host_pin_recovery::Witness>, ) -> Result<(), CandidateError> { let capacity = engine.guest().bridge_intent().map_or(0, |v| v.slots); if selection.version != 1 @@ -67,6 +75,7 @@ fn validate_selection( if predecessor_owner( receipt, selection.previous_boot.as_deref().ok_or_else(invalid)?, + host_pin, )? != *expected { return Err(invalid()); @@ -183,17 +192,39 @@ fn pinned_predecessor_eligible(receipt: &Receipt) -> bool { }) } -fn predecessor_owner(receipt: &Receipt, previous: &str) -> Result { +fn predecessor_owner( + receipt: &Receipt, + previous: &str, + host_pin: Option<&super::super::host_pin_recovery::Witness>, +) -> Result { let startup = receipt .relay_startup .as_ref() .filter(|_| pinned_predecessor_eligible(receipt)) .ok_or_else(invalid)?; - let pin = crate::provider::relay_owner::publication::PinnedEndpoint::load( - &startup.control_root, - super::super::host_relay::context(&receipt.owner, previous)?, - )?; - pin.verify_dead()?; + let context = super::super::host_relay::context(&receipt.owner, previous)?; + let pin = if let Some(selected) = host_pin { + if !selected.matches_graph(receipt) || selected.control_root() != startup.control_root { + return Err(invalid()); + } + let pin = crate::provider::relay_owner::publication::PinnedEndpoint::load_legacy_recovery( + &startup.control_root, + context, + selected.rebind(), + selected.host_boot_micros(), + )?; + if &pin.legacy_summary() != selected.control() { + return Err(invalid()); + } + pin + } else { + let pin = crate::provider::relay_owner::publication::PinnedEndpoint::load( + &startup.control_root, + context, + )?; + pin.verify_dead()?; + pin + }; Ok(pin .fingerprint() .iter() @@ -208,6 +239,7 @@ pub(crate) fn capture_previous_boot( engine: &Engine<'_>, receipt: &Receipt, previous: &str, + host_pin: Option<&super::super::host_pin_recovery::Witness>, ) -> Result { let store = strict_store(candidate, engine)?; let mut selection = Selection { @@ -271,22 +303,23 @@ pub(crate) fn capture_previous_boot( }) .collect(); } else { - selection.predecessor_owner = Some(predecessor_owner(receipt, previous)?); + selection.predecessor_owner = Some(predecessor_owner(receipt, previous, host_pin)?); } } - validate_selection(engine, receipt, &selection)?; + validate_selection_recovery(engine, receipt, &selection, host_pin)?; Ok(selection) } /// Pending recovery may resume only the exact remaining reservations. Released /// slots may be absent; a replacement or new reservation is never adopted. -pub(crate) fn verify_remaining( +pub(crate) fn verify_remaining_recovery( candidate: &Candidate, engine: &Engine<'_>, receipt: &Receipt, selection: &Selection, + host_pin: Option<&super::super::host_pin_recovery::Witness>, ) -> Result<(), CandidateError> { - validate_selection(engine, receipt, selection)?; + validate_selection_recovery(engine, receipt, selection, host_pin)?; if selection.previous_boot.is_none() { return Ok(()); } @@ -529,19 +562,28 @@ pub(crate) fn read( engine: &Engine<'_>, receipt: &Receipt, root: &std::path::Path, +) -> Result { + read_recovery(engine, receipt, root, None) +} +pub(crate) fn read_recovery( + engine: &Engine<'_>, + receipt: &Receipt, + root: &std::path::Path, + host_pin: Option<&super::super::host_pin_recovery::Witness>, ) -> Result { let selection = state::read_bounded(&selection_path(root)?, 65536)?; - validate_selection(engine, receipt, &selection)?; + validate_selection_recovery(engine, receipt, &selection, host_pin)?; Ok(selection) } -pub(crate) fn verify( +pub(crate) fn verify_recovery( candidate: &Candidate, engine: &Engine<'_>, receipt: &Receipt, selection: &Selection, + host_pin: Option<&super::super::host_pin_recovery::Witness>, ) -> Result<(), CandidateError> { - validate_selection(engine, receipt, selection)?; + validate_selection_recovery(engine, receipt, selection, host_pin)?; let store = strict_store(candidate, engine)?; if store.next_launch_serial < selection.serial || store.slots.values().any(|a| a.run == receipt.run) @@ -557,6 +599,16 @@ pub(crate) fn verify( Ok(()) } +#[cfg(test)] +pub(crate) fn verify( + candidate: &Candidate, + engine: &Engine<'_>, + receipt: &Receipt, + selection: &Selection, +) -> Result<(), CandidateError> { + verify_recovery(candidate, engine, receipt, selection, None) +} + #[cfg(test)] mod tests { use super::*; diff --git a/packages/runtime-core/src/provider/graph/dead_owner_cleanup.rs b/packages/runtime-core/src/provider/graph/dead_owner_cleanup.rs index f2d1a8135..9e32a865f 100644 --- a/packages/runtime-core/src/provider/graph/dead_owner_cleanup.rs +++ b/packages/runtime-core/src/provider/graph/dead_owner_cleanup.rs @@ -4,6 +4,7 @@ use super::*; use crate::provider::{identity, lifecycle, state::Owner}; use sha2::{Digest, Sha256}; +use std::os::unix::fs::MetadataExt; const FILE: &str = "dead-owner-cleanup.json"; #[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -322,18 +323,98 @@ pub fn retire_recovered_publisher( } let bridges = intent.bridges.as_ref().ok_or_else(refused)?; bridges::cleanup::verify_recovery_file(&root, bridges, intent.prior_bridges.as_ref())?; + let legacy = super::host_pin_recovery::load_witness(candidate, run)?.filter(|witness| { + witness.graph_sha256() == intent.original_sha256 + && witness.publisher_sha256() == intent.owner_sha256 + && witness.matches_graph(&intent.original) + }); if intent.new_boot.as_deref() == Some(engine.guest().boot_id()) { - host_relay::inspect_cleanup(candidate, &engine, &receipt, false, &environment, bridges)?; - foreground::retire_publisher_path(candidate, run, &intent.owner_sha256, &complete)?; + if let Some(witness) = legacy { + if let Some(retired) = foreground::transport::Retired::acquire(candidate, run)? { + let control_root = witness.control_root().join("relay-control"); + let control_lock = state::Lock::acquire_existing(&control_root)?; + host_relay::inspect_cleanup_recovery( + candidate, + &engine, + &receipt, + false, + &environment, + bridges, + Some(&witness), + )?; + let lock_path = fs::symlink_metadata(control_root.join("operation.lock")) + .map_err(|_| refused())?; + if (lock_path.dev(), lock_path.ino()) != control_lock.identity()? { + return Err(refused()); + } + retired.verify_recovery_with_rebind( + candidate, + run, + &intent.owner_sha256, + &complete, + Some(witness.rebind()), + )?; + } else { + let foreground_root = foreground::transport::root(candidate, run)?; + let foreground_lock = state::Lock::acquire_existing(&foreground_root)?; + let guard = super::host_pin_recovery::acquire_for_cleanup( + candidate, + run, + &engine, + super::host_pin_recovery::CleanupProof { + original: &intent.original, + sha256: &intent.original_sha256, + current_is_original: false, + allow_absent_reservation: true, + publisher_may_be_partial: true, + }, + witness, + )?; + host_relay::inspect_cleanup_recovery( + candidate, + &engine, + &receipt, + false, + &environment, + bridges, + Some(guard.witness()), + )?; + guard.verify_lock()?; + let lock_path = fs::symlink_metadata(foreground_root.join("operation.lock")) + .map_err(|_| refused())?; + if (lock_path.dev(), lock_path.ino()) != foreground_lock.identity()? { + return Err(refused()); + } + foreground::transport::retire_recovered_publisher_locked( + candidate, + run, + &intent.owner_sha256, + &complete, + Some(guard.witness().rebind()), + &foreground_lock, + )?; + } + } else { + host_relay::inspect_cleanup( + candidate, + &engine, + &receipt, + false, + &environment, + bridges, + )?; + foreground::retire_publisher_path(candidate, run, &intent.owner_sha256, &complete)?; + } } else { // A later VM boot has a new bridge registry generation. Recheck the // retained graph and immutable prior proof, then require the publisher // to have been fully retired under the original recovery boot. - foreground::verify_recovered_publisher_retired( + foreground::transport::verify_recovered_publisher_retired_recovery( candidate, run, &intent.owner_sha256, &complete, + legacy.as_ref().map(|witness| witness.rebind()), )?; } Ok(json!({"run":run,"publisher_retired":true,"data_retained":true})) @@ -427,7 +508,13 @@ fn fresh_boot( } fn execute(candidate: &Candidate, run: &str, expected: &str) -> Result { let engine = Engine::connect_cleanup_wait(candidate)?; - let dead = foreground::DeadOwner::acquire(candidate, run)?; + let selected_pin = super::host_pin_recovery::selected_for_old_publisher(candidate, run)?; + let dead = foreground::DeadOwner::acquire_recovery( + candidate, + run, + selected_pin.as_ref().map(|pin| pin.rebind()), + selected_pin.as_ref().map(|pin| pin.host_boot_micros()), + )?; let (receipt, root) = load(candidate, &engine, run)?; if exists(&root.join("one-off-normalization.json"))? { let intent: Intent = state::read(&root.join(FILE))?; @@ -448,6 +535,36 @@ fn execute(candidate: &Candidate, run: &str, expected: &str) -> Result = if exists(&root.join(FILE))? { + Some(state::read(&root.join(FILE))?) + } else { + None + }; + let pin_guard = if let Some(witness) = selected_pin { + let original = existing_intent + .as_ref() + .map_or(&receipt, |intent| &intent.original); + let original_sha = existing_intent + .as_ref() + .map_or(expected, |intent| intent.original_sha256.as_str()); + Some(super::host_pin_recovery::acquire_for_cleanup( + candidate, + run, + &engine, + super::host_pin_recovery::CleanupProof { + original, + sha256: original_sha, + current_is_original: existing_intent.is_none(), + allow_absent_reservation: existing_intent + .as_ref() + .is_some_and(|intent| intent.complete_sha256.is_some()), + publisher_may_be_partial: false, + }, + witness, + )?) + } else { + None + }; let mut intent: Intent = if exists(&root.join(FILE))? { state::read(&root.join(FILE))? } else { @@ -479,8 +596,13 @@ fn execute(candidate: &Candidate, run: &str, expected: &str) -> Result Result Result Result Result Result Result Result Result { } } fn read(path: &Path) -> Result { + read_with_bytes(path).map(|(record, _)| record) +} +fn read_with_bytes(path: &Path) -> Result<(Record, Vec), CandidateError> { let mut file = OpenOptions::new() .read(true) .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) @@ -70,7 +74,121 @@ fn read(path: &Path) -> Result { if bytes.len() as u64 != m.len() { return Err(refused()); } - serde_json::from_slice(&bytes).map_err(|_| refused()) + let record = serde_json::from_slice(&bytes).map_err(|_| refused())?; + Ok((record, bytes)) +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct LegacyReservation { + pub record_sha256: String, + pub sockets: BTreeMap, + pub process: ProcessIdentity, +} +pub(super) fn inspect_legacy( + candidate: &Candidate, + run: &str, + rebind: DeviceRebind, + host_boot_micros: u64, +) -> Result, CandidateError> { + let owner = state::Owner::load(candidate)?; + let directory = root(candidate); + let values = records( + &directory, + &owner.token, + owner.dependency_sockets.map_or(0, |v| v.slots), + )?; + let Some(record) = values.iter().find(|r| r.run == run) else { + return Ok(None); + }; + let (again, bytes) = read_with_bytes(&directory.join(format!("{run}.json")))?; + if &again != record || record.sockets.len() != record.slots.len() { + return Err(refused()); + } + // SAFETY: geteuid has no arguments or side effects. + identity::verify( + &record.process, + &record.process, + &record.process.executable, + unsafe { libc::geteuid() }, + )?; + rebind.definitely_dead_before_boot(&record.process, host_boot_micros)?; + for slot in record.slots.values() { + let expected = record.sockets.get(slot).ok_or_else(refused)?; + let path = socket(&owner.short_home, *slot); + let metadata = fs::symlink_metadata(&path).map_err(|_| refused())?; + if !metadata.file_type().is_socket() + || metadata.uid() != record.process.uid + || metadata.mode() & 0o7777 != 0o600 + || metadata.nlink() != 1 + || !rebind.matches(*expected, (metadata.dev(), metadata.ino())) + { + return Err(refused()); + } + crate::provider::relay_owner::publication::dead::no_listener(&path) + .map_err(|_| refused())?; + } + Ok(Some(LegacyReservation { + record_sha256: format!("{:x}", Sha256::digest(bytes)), + sockets: record.sockets.clone(), + process: record.process.clone(), + })) +} +pub(super) fn verify_legacy_remaining( + candidate: &Candidate, + run: &str, + rebind: DeviceRebind, + selected: &LegacyReservation, + allow_absent_record: bool, +) -> Result<(), CandidateError> { + let owner = state::Owner::load(candidate)?; + let path = root(candidate).join(format!("{run}.json")); + if absent(&path)? { + if !allow_absent_record { + return Err(refused()); + } + for slot in selected.sockets.keys() { + if !absent(&socket(&owner.short_home, *slot))? { + return Err(refused()); + } + } + return Ok(()); + } + let values = records( + &root(candidate), + &owner.token, + owner.dependency_sockets.map_or(0, |v| v.slots), + )?; + let record = values + .iter() + .find(|record| record.run == run) + .ok_or_else(refused)?; + let (same, bytes) = read_with_bytes(&path)?; + if &same != record + || selected.record_sha256 != format!("{:x}", Sha256::digest(bytes)) + || selected.sockets != record.sockets + || selected.process != record.process + { + return Err(refused()); + } + for slot in record.slots.values() { + let path = socket(&owner.short_home, *slot); + if absent(&path)? { + continue; + } + let expected = record.sockets.get(slot).ok_or_else(refused)?; + let observed = fs::symlink_metadata(&path).map_err(|_| refused())?; + if !observed.file_type().is_socket() + || observed.uid() != record.process.uid + || observed.mode() & 0o7777 != 0o600 + || observed.nlink() != 1 + || !rebind.matches(*expected, (observed.dev(), observed.ino())) + { + return Err(refused()); + } + crate::provider::relay_owner::publication::dead::no_listener(&path) + .map_err(|_| refused())?; + } + Ok(()) } fn valid(record: &Record, owner: &str, capacity: u8) -> bool { record.version == 1 @@ -339,7 +457,11 @@ pub fn inspect(candidate: &Candidate) -> Result Result<(), CandidateError> { +fn recover_record( + candidate: &Candidate, + record: &Record, + rebind: Option, +) -> Result<(), CandidateError> { if identity::alive(record.process.pid)? { return Err(refused()); } @@ -347,10 +469,14 @@ fn recover_record(candidate: &Candidate, record: &Record) -> Result<(), Candidat if record.owner != owner.token { return Err(refused()); } - recover_paths(&owner.short_home, record)?; + recover_paths(&owner.short_home, record, rebind)?; remove_record(&root(candidate), record) } -fn recover_paths(home: &Path, record: &Record) -> Result<(), CandidateError> { +fn recover_paths( + home: &Path, + record: &Record, + rebind: Option, +) -> Result<(), CandidateError> { // Validate every selected path before removing any. Unexpected or unrecorded // paths are never adopted, even when nobody currently listens on them. let mut selected = Vec::new(); @@ -364,7 +490,11 @@ fn recover_paths(home: &Path, record: &Record) -> Result<(), CandidateError> { || m.mode() & 0o7777 != 0o600 || m.uid() != record.process.uid || m.nlink() != 1 - || record.sockets.get(slot) != Some(&(m.dev(), m.ino())) + || !record.sockets.get(slot).is_some_and(|expected| { + rebind.map_or(*expected == (m.dev(), m.ino()), |v| { + v.matches(*expected, (m.dev(), m.ino())) + }) + }) { return Err(refused()); } @@ -402,13 +532,14 @@ pub fn recover_orphan( if fingerprint(record)? != expected || !absent(&super::directory(candidate, run)?)? { return Err(refused()); } - recover_record(candidate, record)?; + recover_record(candidate, record, None)?; Ok(serde_json::json!({"run":run,"released":true})) } /// Caller holds Engine lease and has completed dead-owner cleanup/verification. pub(super) fn recover_cleaned( candidate: &Candidate, receipt: &Receipt, + rebind: Option<(DeviceRebind, &LegacyReservation)>, ) -> Result<(), CandidateError> { let directory = root(candidate); if absent(&directory.join(format!("{}.json", receipt.run)))? { @@ -438,7 +569,17 @@ pub(super) fn recover_cleaned( { return Err(refused()); } - recover_record(candidate, record) + if let Some((_, selected)) = rebind { + let (again, bytes) = read_with_bytes(&directory.join(format!("{}.json", receipt.run)))?; + if &again != record + || selected.record_sha256 != format!("{:x}", Sha256::digest(bytes)) + || selected.sockets != record.sockets + || selected.process != record.process + { + return Err(refused()); + } + } + recover_record(candidate, record, rebind.map(|(value, _)| value)) } #[cfg(test)] @@ -555,14 +696,14 @@ mod tests { let mut b = fixture_record('2', 1); let a_listener = listener(&fixture.0, &mut a, 0); let b_listener = listener(&fixture.0, &mut b, 1); - assert!(recover_paths(&fixture.0, &a).is_err()); + assert!(recover_paths(&fixture.0, &a, None).is_err()); drop(a_listener); let old = a.sockets[&0]; a.sockets.insert(0, (old.0, old.1 + 1)); - assert!(recover_paths(&fixture.0, &a).is_err()); + assert!(recover_paths(&fixture.0, &a, None).is_err()); assert!(socket(&fixture.0, 0).exists()); a.sockets.insert(0, old); - recover_paths(&fixture.0, &a).unwrap(); + recover_paths(&fixture.0, &a, None).unwrap(); assert!(!socket(&fixture.0, 0).exists()); assert!(UnixStream::connect(socket(&fixture.0, 1)).is_ok()); drop(b_listener); @@ -575,10 +716,10 @@ mod tests { let first = listener(&fixture.0, &mut a, 0); let second = listener(&fixture.0, &mut a, 1); drop(first); - assert!(recover_paths(&fixture.0, &a).is_err()); + assert!(recover_paths(&fixture.0, &a, None).is_err()); assert!(socket(&fixture.0, 0).exists()); drop(second); - recover_paths(&fixture.0, &a).unwrap(); + recover_paths(&fixture.0, &a, None).unwrap(); assert!(!socket(&fixture.0, 0).exists()); assert!(!socket(&fixture.0, 1).exists()); } diff --git a/packages/runtime-core/src/provider/graph/foreground.rs b/packages/runtime-core/src/provider/graph/foreground.rs index 7f46e84be..366769dc6 100644 --- a/packages/runtime-core/src/provider/graph/foreground.rs +++ b/packages/runtime-core/src/provider/graph/foreground.rs @@ -20,7 +20,6 @@ mod tests; pub(in crate::provider::graph) mod transport; pub(in crate::provider::graph) use transport::DeadOwner; pub(in crate::provider::graph) use transport::retire_recovered_publisher as retire_publisher_path; -pub(in crate::provider::graph) use transport::verify_recovered_publisher_retired; use transport::{Publication, WireRequest}; fn refused() -> CandidateError { CandidateError::new( diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test.rs b/packages/runtime-core/src/provider/graph/foreground/native_test.rs index 016db1b0f..aba7ac312 100644 --- a/packages/runtime-core/src/provider/graph/foreground/native_test.rs +++ b/packages/runtime-core/src/provider/graph/foreground/native_test.rs @@ -542,5 +542,6 @@ mod dependency_rebind; mod dependency_slots; mod startup_cancellation; +mod host_pin_recovery; mod retired_recovery_cleanup; mod same_boot_recovery; diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test/host_pin_recovery.rs b/packages/runtime-core/src/provider/graph/foreground/native_test/host_pin_recovery.rs new file mode 100644 index 000000000..1ce87e513 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/foreground/native_test/host_pin_recovery.rs @@ -0,0 +1,537 @@ +//! Real CLI path for explicitly witnessed legacy host-pin cleanup. An isolated +//! test VM supplies the guest; only fixture receipt device numbers are changed. +use super::dependency_rebind::{checked_cli, exec, snapshot}; +use super::*; +use std::{ + io::Write, + os::unix::fs::{MetadataExt, OpenOptionsExt}, +}; + +fn ready(owner: &mut Process, run: &str, deadline: Instant) { + loop { + assert!( + owner.poll().is_none(), + "foreground owner exited before ready" + ); + if let Some(end) = owner.out.iter().position(|byte| *byte == b'\n') { + let value: Value = serde_json::from_slice(&owner.out[..end]).unwrap(); + assert_eq!(value["kind"], "graph_foreground_ready"); + assert_eq!(value["run"], run); + return; + } + assert!(Instant::now() < deadline, "foreground readiness deadline"); + std::thread::sleep(Duration::from_millis(10)); + } +} + +fn legacy_pin(path: &Path, old: u64, before_boot: u64, control: bool) -> Vec { + let mut value: Value = state::read(path).unwrap(); + for key in if control { + ["parent", "endpoint"] + } else { + ["parent", "socket"] + } { + let pair = value[key].as_array_mut().unwrap(); + assert_ne!(pair[0].as_u64().unwrap(), old); + pair[0] = json!(old); + } + // This is a synthetic previous-physical-boot receipt. The actual owner + // exited before the test changes its private fixture bytes. + value["process"]["pid"] = json!(i32::MAX); + value["process"]["start_micros"] = json!(before_boot - 1); + state::write(path, &value).unwrap(); + fs::read(path).unwrap() +} + +fn refused_cli( + binary: &Path, + candidate: &Candidate, + args: &[&str], + expected_code: &str, + deadline: Instant, +) { + let mut process = Process::start(binary, candidate, args, None); + let status = process.wait(deadline); + assert_eq!( + status.code(), + Some(2), + "expected a CLI refusal, not a crash" + ); + assert!(process.out.is_empty(), "refused action emitted a result"); + let error: Value = serde_json::from_slice(&process.err).expect("structured CLI error"); + assert_eq!(error["code"], expected_code); + assert!(error["message"].as_str().is_some_and(|v| !v.is_empty())); +} + +fn publish_args<'a>(run: &'a str, expected: &'a str) -> [&'a str; 8] { + [ + "graph", + "recover-host-pins", + "--run-id", + run, + "--expect-selection", + expected, + "--accept-legacy-device-rebind", + "--json", + ] +} + +struct TemporarilyMoved { + original: PathBuf, + moved: PathBuf, +} +impl TemporarilyMoved { + fn new(original: PathBuf) -> Self { + let moved = original.with_extension("host-pin-test-held"); + assert!(!moved.exists()); + fs::rename(&original, &moved).unwrap(); + Self { original, moved } + } +} +impl Drop for TemporarilyMoved { + fn drop(&mut self) { + if self.moved.exists() { + assert!(!self.original.exists(), "foreign replacement at held path"); + fs::rename(&self.moved, &self.original).unwrap(); + } + } +} + +/// Preserve exact pre-mutation bytes outside managed state. A killed test can +/// be recovered from this private fixture artifact; normal unwinding restores +/// only the bytes this test itself wrote, never a foreign replacement. +struct ProviderReceiptRestore { + path: PathBuf, + original: Vec, + current: Vec, +} +impl ProviderReceiptRestore { + fn new(candidate: &Candidate, path: PathBuf) -> Self { + let original = fs::read(&path).unwrap(); + let backup = candidate + .checkout + .join("host-pin-test-provider-owner-original.json"); + let mut artifact = fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&backup) + .unwrap(); + artifact.write_all(&original).unwrap(); + artifact.sync_all().unwrap(); + fs::File::open(&candidate.checkout) + .unwrap() + .sync_all() + .unwrap(); + Self { + path, + current: original.clone(), + original, + } + } + fn replace(&mut self, bytes: Vec) { + assert_eq!(fs::read(&self.path).unwrap(), self.current); + fs::write(&self.path, &bytes).unwrap(); + self.current = bytes; + } + fn restore(&mut self) { + assert_eq!(fs::read(&self.path).unwrap(), self.current); + fs::write(&self.path, &self.original).unwrap(); + assert_eq!(fs::read(&self.path).unwrap(), self.original); + self.current = self.original.clone(); + } +} +impl Drop for ProviderReceiptRestore { + fn drop(&mut self) { + if self.current != self.original + && fs::read(&self.path).ok().as_deref() == Some(self.current.as_slice()) + { + let _ = fs::write(&self.path, &self.original); + } + } +} + +#[test] +#[ignore = "Owned capacity-two native VM, pinned static image and external 300s watchdog required"] +fn explicit_host_pin_witness_recovers_only_selected_previous_boot_run() { + let deadline = Instant::now() + Duration::from_secs(270); + let candidate = + Candidate::discover(Path::new(&std::env::var("HACK_LOCAL_TEST_ROOT").unwrap())).unwrap(); + let binary = PathBuf::from(std::env::var("HACK_LOCAL_TEST_BINARY").unwrap()); + let image = std::env::var("HACK_LOCAL_TEST_IMAGE").unwrap(); + let fixtures = [graph::tests::Fixture::new(), graph::tests::Fixture::new()]; + let runs = [ + graph::probes::token().unwrap(), + graph::probes::token().unwrap(), + ]; + let selection_root = graph::tests::Fixture::new(); + let mut plans = Vec::new(); + let mut selections = Vec::new(); + let mut dependencies = Vec::new(); + for (index, fixture) in fixtures.iter().enumerate() { + state::write(&fixture.0.join("compose.yaml"), &json!({ + "services":{"web":{"image":image,"read_only":true,"network_mode":"none", + "init":true,"user":"0:0","entrypoint":["/bin/sleep","300"],"command":[], + "volumes":["data:/data"],"healthcheck":{"test":["CMD","/bin/hack-graph-startup-app","complete"], + "interval":"200ms","timeout":"2s","retries":10,"start_period":"500ms"}}}, + "volumes":{"data":{}} + })).unwrap(); + let review = project::plan( + &candidate, + project::PlanOptions { + branch: None, + project: &fixture.0, + compose_file: Path::new("compose.yaml"), + profiles: &[], + }, + ) + .unwrap(); + let selection = selection_root.0.join(format!("dependencies-{index}.json")); + state::write(&selection, &json!({"version":1,"plan":review.plan_id, + "artifact":"/tmp/unused-control-only-artifact","artifact_sha256":"a".repeat(64),"dependencies":[]})).unwrap(); + let dependency = checked_cli( + "dependency-plan", + &binary, + &candidate, + &[ + "graph", + "dependency-plan", + "--dependencies", + selection.to_str().unwrap(), + "--json", + ], + deadline, + ); + plans.push(review); + selections.push(selection); + dependencies.push(dependency); + } + let start = |index: usize, generation: Option<&str>| { + let normalized = fixtures[index].0.join("compose.yaml"); + let mut args = vec![ + "graph", + if generation.is_some() { + "serve-restore" + } else { + "serve" + }, + "--project", + fixtures[index].0.to_str().unwrap(), + "--file", + "compose.yaml", + "--expect-plan", + &plans[index].plan_id, + "--run-id", + &runs[index], + "--ready", + "web=healthy", + "--timeout-seconds", + "90", + "--dependencies", + selections[index].to_str().unwrap(), + "--expect-dependencies", + dependencies[index]["dependency_plan_id"].as_str().unwrap(), + "--normalized-file", + normalized.to_str().unwrap(), + "--expect-original", + &plans[index].plan.compose_sha256, + "--expect-namespace", + &plans[index].plan.namespace, + "--json", + ]; + if let Some(generation) = generation { + args.extend(["--expect-generation", generation]); + } + Process::start(&binary, &candidate, &args, None) + }; + let mut selected_owner = start(0, None); + let mut selected_cleanup = Cleanup { + binary: &binary, + candidate: &candidate, + run: &runs[0], + done: false, + }; + ready(&mut selected_owner, &runs[0], deadline); + assert_eq!( + exec(&binary, &candidate, &runs[0], "web", "write-data", deadline)["exit_code"], + 0 + ); + let original = snapshot(&candidate, &runs[0], deadline).receipt; + assert_eq!(original.phase, "ready-observed"); + let expected_receipt = format!( + "{:x}", + Sha256::digest(serde_json::to_vec_pretty(&original).unwrap()) + ); + let old_boot = crate::provider::lifecycle::status(&candidate) + .unwrap() + .guest_boot_id + .unwrap(); + selected_owner.child.kill().unwrap(); + selected_owner.wait(deadline); + assert_eq!( + crate::provider::lifecycle::down(&candidate).unwrap().phase, + "stopped" + ); + let new_boot = crate::provider::lifecycle::up(&candidate) + .unwrap() + .guest_boot_id + .unwrap(); + assert_ne!(old_boot, new_boot); + + let mut sibling_owner = start(1, None); + let mut sibling_cleanup = Cleanup { + binary: &binary, + candidate: &candidate, + run: &runs[1], + done: false, + }; + ready(&mut sibling_owner, &runs[1], deadline); + assert_eq!( + exec(&binary, &candidate, &runs[1], "web", "write-data", deadline)["exit_code"], + 0 + ); + let sibling = serde_json::to_vec(&snapshot(&candidate, &runs[1], deadline).receipt).unwrap(); + + let root = graph::directory(&candidate, &runs[0]).unwrap(); + let graph_bytes = fs::read(root.join("state.json")).unwrap(); + let startup = original.relay_startup.as_ref().unwrap(); + let publisher_root = transport::root(&candidate, &runs[0]).unwrap(); + let publisher = publisher_root.join("owner.json"); + let control = startup.control_root.join("relay-control/owner.json"); + let current = fs::symlink_metadata(&publisher_root).unwrap().dev(); + let old = current.checked_add(1).unwrap(); + let before_boot = crate::provider::lifecycle::host_filesystem::host_boot_micros().unwrap(); + let publisher_bytes = legacy_pin(&publisher, old, before_boot, false); + let control_bytes = legacy_pin(&control, old, before_boot, true); + + // Ordinary recovery remains strict; the new witness is required to use + // the legacy device-only pins and is scoped to the selected run. + assert!(graph::foreground::transport::Pin::load(&candidate, &runs[0]).is_err()); + let inspected = checked_cli( + "inspect-host-pins", + &binary, + &candidate, + &[ + "graph", + "inspect-host-pin-recovery", + "--run-id", + &runs[0], + "--json", + ], + deadline, + ); + let selection = inspected["selection_sha256"].as_str().unwrap(); + assert_eq!(selection.len(), 64); + assert_eq!(inspected["old_device"], old); + assert_eq!(inspected["new_device"], current); + assert_eq!(fs::read(root.join("state.json")).unwrap(), graph_bytes); + assert_eq!(fs::read(&publisher).unwrap(), publisher_bytes); + assert_eq!(fs::read(&control).unwrap(), control_bytes); + + let witness_path = root.join("host-pin-recovery.json"); + let inspect_args = [ + "graph", + "inspect-host-pin-recovery", + "--run-id", + &runs[0], + "--json", + ]; + // A stale selection must not publish a witness or rewrite any source + // receipt. The three temporary path removals also preserve exact bytes. + refused_cli( + &binary, + &candidate, + &publish_args(&runs[0], &"0".repeat(64)), + "graph_host_pin_recovery", + deadline, + ); + assert!(!witness_path.exists()); + assert_eq!(fs::read(root.join("state.json")).unwrap(), graph_bytes); + assert_eq!(fs::read(&publisher).unwrap(), publisher_bytes); + assert_eq!(fs::read(&control).unwrap(), control_bytes); + { + let _missing = TemporarilyMoved::new(publisher_root.clone()); + refused_cli( + &binary, + &candidate, + &inspect_args, + "provider_state", + deadline, + ); + assert!(!witness_path.exists()); + } + assert_eq!(fs::read(&publisher).unwrap(), publisher_bytes); + { + let _missing = TemporarilyMoved::new(startup.control_root.join("relay-control")); + refused_cli( + &binary, + &candidate, + &inspect_args, + "provider_state", + deadline, + ); + assert!(!witness_path.exists()); + } + assert_eq!(fs::read(&control).unwrap(), control_bytes); + + // The provider receipt is valid JSON throughout. A changed raw byte + // selection and a changed guest boot must both refuse without writes. + let provider_path = candidate.state_root.join("run/smolvm/owner.json"); + let mut owner_restore = ProviderReceiptRestore::new(&candidate, provider_path.clone()); + let provider_bytes = owner_restore.original.clone(); + let mut whitespace = provider_bytes.clone(); + whitespace.push(b' '); + owner_restore.replace(whitespace.clone()); + refused_cli( + &binary, + &candidate, + &publish_args(&runs[0], selection), + "graph_host_pin_recovery", + deadline, + ); + assert_eq!(fs::read(&provider_path).unwrap(), whitespace); + assert!(!witness_path.exists()); + owner_restore.restore(); + let mut changed_boot: Value = serde_json::from_slice(&provider_bytes).unwrap(); + changed_boot["guest_boot_id"] = json!("f".repeat(32)); + assert_ne!(changed_boot["guest_boot_id"], new_boot); + owner_restore.replace(serde_json::to_vec_pretty(&changed_boot).unwrap()); + let boot_bytes = fs::read(&provider_path).unwrap(); + refused_cli( + &binary, + &candidate, + &inspect_args, + "graph_host_pin_recovery", + deadline, + ); + assert_eq!(fs::read(&provider_path).unwrap(), boot_bytes); + assert!(!witness_path.exists()); + owner_restore.restore(); + assert_eq!(fs::read(root.join("state.json")).unwrap(), graph_bytes); + assert_eq!(fs::read(&publisher).unwrap(), publisher_bytes); + assert_eq!(fs::read(&control).unwrap(), control_bytes); + + let refreshed = checked_cli( + "inspect-after-negative-controls", + &binary, + &candidate, + &inspect_args, + deadline, + ); + let selection = refreshed["selection_sha256"].as_str().unwrap(); + let recovered = checked_cli( + "publish-host-pin-witness", + &binary, + &candidate, + &publish_args(&runs[0], selection), + deadline, + ); + assert_eq!(recovered["witness_published"], true); + assert_eq!(fs::read(root.join("state.json")).unwrap(), graph_bytes); + assert_eq!(fs::read(&publisher).unwrap(), publisher_bytes); + assert_eq!(fs::read(&control).unwrap(), control_bytes); + assert_eq!( + serde_json::to_vec(&snapshot(&candidate, &runs[1], deadline).receipt).unwrap(), + sibling + ); + + let cleanup = checked_cli( + "legacy-dead-owner-cleanup", + &binary, + &candidate, + &[ + "graph", + "recover-cleanup", + "--run-id", + &runs[0], + "--expect-receipt", + &expected_receipt, + "--json", + ], + deadline, + ); + assert_eq!(cleanup["phase"], "stopped-data-retained"); + let retained = snapshot(&candidate, &runs[0], deadline).receipt; + assert_eq!( + retained.resources["volume:data"].name, + original.resources["volume:data"].name + ); + for _ in 0..2 { + let retired = checked_cli( + "legacy-publisher-retirement", + &binary, + &candidate, + &[ + "graph", + "retire-recovered-publisher", + "--run-id", + &runs[0], + "--expect-owner", + &retained.owner, + "--json", + ], + deadline, + ); + assert_eq!(retired["publisher_retired"], true); + } + let restoration = graph::foreground::restore_selection(&candidate, &runs[0]).unwrap(); + selected_owner = start(0, Some(restoration["generation"].as_str().unwrap())); + ready(&mut selected_owner, &runs[0], deadline); + assert_eq!( + exec(&binary, &candidate, &runs[0], "web", "read-data", deadline)["exit_code"], + 0, + "selected marker must survive cleanup, retirement and same-run restore" + ); + let restored = snapshot(&candidate, &runs[0], deadline).receipt; + assert_eq!( + restored.resources["volume:data"].name, + original.resources["volume:data"].name + ); + assert_ne!( + restored.resources["container:web"].id, + original.resources["container:web"].id + ); + assert!(sibling_owner.poll().is_none()); + assert_eq!( + serde_json::to_vec(&snapshot(&candidate, &runs[1], deadline).receipt).unwrap(), + sibling + ); + assert_eq!( + exec(&binary, &candidate, &runs[1], "web", "read-data", deadline)["exit_code"], + 0 + ); + let selected_removed = checked_cli( + "remove-selected-data", + &binary, + &candidate, + &[ + "graph", + "cleanup", + "--run-id", + &runs[0], + "--remove-data", + "--json", + ], + deadline, + ); + assert_eq!(selected_removed["phase"], "removed"); + assert!(selected_owner.wait(deadline).success()); + selected_cleanup.done = true; + let removed = checked_cli( + "remove-sibling-data", + &binary, + &candidate, + &[ + "graph", + "cleanup", + "--run-id", + &runs[1], + "--remove-data", + "--json", + ], + deadline, + ); + assert_eq!(removed["phase"], "removed"); + assert!(sibling_owner.wait(deadline).success()); + sibling_cleanup.done = true; +} diff --git a/packages/runtime-core/src/provider/graph/foreground/transport.rs b/packages/runtime-core/src/provider/graph/foreground/transport.rs index 8ab4a43f3..2064bfbc6 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport.rs @@ -1,5 +1,6 @@ use super::{Candidate, CandidateError, refused}; use crate::provider::{ + host_pin::DeviceRebind, identity::{self, ProcessIdentity}, state, }; @@ -239,6 +240,7 @@ fn selected_retirement_path( owner: &str, socket: bool, expected: (u64, u64), + device_rebind: Option, ) -> Result<(PathBuf, bool), CandidateError> { let original = root.join(if socket { "control.sock" } else { "owner.json" }); let retired = retired_path(root, owner, socket); @@ -247,8 +249,9 @@ fn selected_retirement_path( (None, Some(found)) => (retired, found, false), _ => return Err(retirement_refused()), }; - if id(¤t) != expected - || !private(¤t) + if !device_rebind.map_or(id(¤t) == expected, |v| { + v.matches(expected, id(¤t)) + }) || !private(¤t) || (socket && !current.file_type().is_socket()) || (!socket && (!current.is_file() || current.nlink() != 1 || current.len() > 8192)) { @@ -260,19 +263,26 @@ fn selected_retirement_path( fn verify_retirement( candidate: &Candidate, run: &str, - expected_owner: &str, - expected_receipt: &str, + expected: (&str, &str), root: &Path, lock: &state::Lock, intent: &Retirement, + device_rebind: Option, ) -> Result<(bool, bool), CandidateError> { + let root_id = id(&fs::symlink_metadata(root).map_err(|_| retirement_refused())?); + let lock_path_id = + id(&fs::symlink_metadata(root.join("operation.lock")).map_err(|_| retirement_refused())?); + let parent_matches = device_rebind.map_or(intent.parent == root_id, |v| { + v.matches(intent.parent, root_id) + }); if intent.version != 1 || intent.candidate != candidate.checkout || intent.run != run - || intent.owner_sha256 != expected_owner - || intent.receipt_sha256 != expected_receipt - || intent.parent != id(&fs::symlink_metadata(root).map_err(|_| retirement_refused())?) + || intent.owner_sha256 != expected.0 + || intent.receipt_sha256 != expected.1 + || !parent_matches || intent.lock != lock.identity().map_err(|_| retirement_refused())? + || lock_path_id != intent.lock || intent.owner.parent != intent.parent || intent.owner.socket != intent.socket || intent.owner.candidate != candidate.checkout @@ -282,9 +292,9 @@ fn verify_retirement( return Err(retirement_refused()); } let (socket_path, socket_original) = - selected_retirement_path(root, expected_owner, true, intent.socket)?; + selected_retirement_path(root, expected.0, true, intent.socket, device_rebind)?; let (record_path, record_original) = - selected_retirement_path(root, expected_owner, false, intent.record)?; + selected_retirement_path(root, expected.0, false, intent.record, None)?; // Socket retirement precedes record retirement. The inverse is foreign. if socket_original && !record_original { return Err(retirement_refused()); @@ -304,7 +314,7 @@ fn verify_retirement( .read_to_end(&mut bytes) .map_err(|_| retirement_refused())?; let record: Record = serde_json::from_slice(&bytes).map_err(|_| retirement_refused())?; - if record != intent.owner || format!("{:x}", Sha256::digest(&bytes)) != expected_owner { + if record != intent.owner || format!("{:x}", Sha256::digest(&bytes)) != expected.0 { return Err(retirement_refused()); } Ok((socket_original, record_original)) @@ -318,6 +328,15 @@ pub(in crate::provider::graph) fn retire_recovered_publisher( run: &str, expected_owner: &str, expected_receipt: &str, +) -> Result<(), CandidateError> { + retire_recovered_publisher_recovery(candidate, run, expected_owner, expected_receipt, None) +} +pub(in crate::provider::graph) fn retire_recovered_publisher_recovery( + candidate: &Candidate, + run: &str, + expected_owner: &str, + expected_receipt: &str, + device_rebind: Option, ) -> Result<(), CandidateError> { if !super::super::hex(expected_owner, 64) || !super::super::hex(expected_receipt, 64) { return Err(retirement_refused()); @@ -325,6 +344,24 @@ pub(in crate::provider::graph) fn retire_recovered_publisher( let root = root(candidate, run)?; state::check_private_directory(&root).map_err(|_| retirement_refused())?; let lock = state::Lock::acquire_existing(&root).map_err(|_| retirement_refused())?; + retire_recovered_publisher_locked( + candidate, + run, + expected_owner, + expected_receipt, + device_rebind, + &lock, + ) +} +pub(in crate::provider::graph) fn retire_recovered_publisher_locked( + candidate: &Candidate, + run: &str, + expected_owner: &str, + expected_receipt: &str, + device_rebind: Option, + lock: &state::Lock, +) -> Result<(), CandidateError> { + let root = root(candidate, run)?; let path = retirement_path(&root, expected_owner); let pending = path.with_extension("pending"); let name = pending @@ -335,7 +372,7 @@ pub(in crate::provider::graph) fn retire_recovered_publisher( let intent: Retirement = if metadata(&path)?.is_some() { state::read(&path).map_err(|_| retirement_refused())? } else { - let pin = Pin::read(candidate, run)?; + let pin = Pin::read_with_rebind(candidate, run, device_rebind)?; if format!("{:x}", Sha256::digest(&pin.bytes)) != expected_owner || identity::alive(pin.record.process.pid).unwrap_or(true) { @@ -360,11 +397,11 @@ pub(in crate::provider::graph) fn retire_recovered_publisher( let (socket_original, record_original) = verify_retirement( candidate, run, - expected_owner, - expected_receipt, + (expected_owner, expected_receipt), &root, - &lock, + lock, &intent, + device_rebind, )?; if socket_original { fs::rename( @@ -380,11 +417,11 @@ pub(in crate::provider::graph) fn retire_recovered_publisher( let _ = verify_retirement( candidate, run, - expected_owner, - expected_receipt, + (expected_owner, expected_receipt), &root, - &lock, + lock, &intent, + device_rebind, )?; fs::rename( root.join("owner.json"), @@ -398,11 +435,11 @@ pub(in crate::provider::graph) fn retire_recovered_publisher( let (socket_original, record_original) = verify_retirement( candidate, run, - expected_owner, - expected_receipt, + (expected_owner, expected_receipt), &root, - &lock, + lock, &intent, + device_rebind, )?; if socket_original || record_original { return Err(retirement_refused()); @@ -413,11 +450,27 @@ pub(in crate::provider::graph) fn retire_recovered_publisher( /** A later boot may resume frontend recovery only after retirement was fully * recorded. This read-only proof never begins or completes a partial move. */ +#[cfg(test)] pub(in crate::provider::graph) fn verify_recovered_publisher_retired( candidate: &Candidate, run: &str, expected_owner: &str, expected_receipt: &str, +) -> Result<(), CandidateError> { + verify_recovered_publisher_retired_recovery( + candidate, + run, + expected_owner, + expected_receipt, + None, + ) +} +pub(in crate::provider::graph) fn verify_recovered_publisher_retired_recovery( + candidate: &Candidate, + run: &str, + expected_owner: &str, + expected_receipt: &str, + device_rebind: Option, ) -> Result<(), CandidateError> { if !super::super::hex(expected_owner, 64) || !super::super::hex(expected_receipt, 64) { return Err(retirement_refused()); @@ -430,11 +483,11 @@ pub(in crate::provider::graph) fn verify_recovered_publisher_retired( let (socket_original, record_original) = verify_retirement( candidate, run, - expected_owner, - expected_receipt, + (expected_owner, expected_receipt), &root, &lock, &intent, + device_rebind, )?; if socket_original || record_original { return Err(retirement_refused()); @@ -485,11 +538,12 @@ fn peer(stream: &UnixStream) -> Result { identity::verify(&observed, &observed, &observed.executable, uid).map_err(|_| refused())?; Ok(observed) } -pub(super) struct Pin { +pub(in crate::provider::graph) struct Pin { root: PathBuf, record: Record, bytes: Vec, record_id: (u64, u64), + device_rebind: Option, } impl Pin { pub fn load(candidate: &Candidate, run: &str) -> Result { @@ -498,6 +552,13 @@ impl Pin { Ok(pin) } fn read(candidate: &Candidate, run: &str) -> Result { + Self::read_with_rebind(candidate, run, None) + } + fn read_with_rebind( + candidate: &Candidate, + run: &str, + device_rebind: Option, + ) -> Result { let root = root(candidate, run)?; state::check_private_directory(&root).map_err(|_| refused())?; let mut file = OpenOptions::new() @@ -530,6 +591,7 @@ impl Pin { record, bytes, record_id: id(&metadata), + device_rebind, }; pin.verify_files()?; Ok(pin) @@ -540,10 +602,10 @@ impl Pin { let socket = fs::symlink_metadata(self.root.join("control.sock")).map_err(|_| refused())?; let file = fs::symlink_metadata(self.root.join("owner.json")).map_err(|_| refused())?; if !parent.is_dir() - || id(&parent) != self.record.parent + || !self.matches_pin(self.record.parent, id(&parent)) || !socket.file_type().is_socket() || !private(&socket) - || id(&socket) != self.record.socket + || !self.matches_pin(self.record.socket, id(&socket)) || !file.is_file() || !private(&file) || file.nlink() != 1 @@ -570,6 +632,46 @@ impl Pin { } Ok(()) } + fn matches_pin(&self, recorded: (u64, u64), observed: (u64, u64)) -> bool { + self.device_rebind + .map_or(recorded == observed, |v| v.matches(recorded, observed)) + } + pub(in crate::provider::graph) fn legacy_summary( + candidate: &Candidate, + run: &str, + device_rebind: DeviceRebind, + host_boot_micros: u64, + ) -> Result { + let pin = Self::read_with_rebind(candidate, run, Some(device_rebind))?; + // SAFETY: geteuid has no arguments or side effects. + identity::verify( + &pin.record.process, + &pin.record.process, + &pin.record.process.executable, + unsafe { libc::geteuid() }, + )?; + device_rebind.definitely_dead_before_boot(&pin.record.process, host_boot_micros)?; + no_listener(&pin.root.join("control.sock"))?; + Ok(LegacyPublisher { + owner_sha256: format!("{:x}", Sha256::digest(&pin.bytes)), + parent: pin.record.parent, + socket: pin.record.socket, + record: pin.record_id, + process: pin.record.process, + }) + } + pub(in crate::provider::graph) fn legacy_recorded_device( + candidate: &Candidate, + run: &str, + ) -> Result { + let root = root(candidate, run)?; + state::check_private_directory(&root)?; + let record: Record = state::read_bounded(&root.join("owner.json"), 8192)?; + if record.version != 1 || record.candidate != candidate.checkout || record.run != run { + return Err(refused()); + } + Ok(record.parent.0) + } pub fn verify(&self) -> Result<(), CandidateError> { self.verify_files()?; let expected = &self.record.process; @@ -647,6 +749,16 @@ impl Pin { Ok(stream) } } + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(in crate::provider::graph) struct LegacyPublisher { + pub owner_sha256: String, + pub parent: (u64, u64), + pub socket: (u64, u64), + pub record: (u64, u64), + pub process: ProcessIdentity, +} pub(super) struct Publication { listener: UnixListener, pin: Pin, @@ -706,6 +818,7 @@ impl Publication { record, bytes, record_id, + device_rebind: None, }; pin.verify()?; Ok(Self { @@ -980,6 +1093,16 @@ impl Retired { run: &str, owner_sha256: &str, complete_sha256: &str, + ) -> Result<(), CandidateError> { + self.verify_recovery_with_rebind(candidate, run, owner_sha256, complete_sha256, None) + } + pub fn verify_recovery_with_rebind( + &self, + candidate: &Candidate, + run: &str, + owner_sha256: &str, + complete_sha256: &str, + device_rebind: Option, ) -> Result<(), CandidateError> { self.verify()?; if !super::super::hex(owner_sha256, 64) @@ -996,11 +1119,11 @@ impl Retired { let (socket_original, record_original) = verify_retirement( candidate, run, - owner_sha256, - complete_sha256, + (owner_sha256, complete_sha256), &self.root, &self._lock, &intent, + device_rebind, )?; if socket_original || record_original { return Err(retirement_refused()); @@ -1018,16 +1141,39 @@ impl DeadOwner { pub(in crate::provider::graph) fn acquire( candidate: &Candidate, run: &str, + ) -> Result { + Self::acquire_recovery(candidate, run, None, None) + } + pub(in crate::provider::graph) fn acquire_recovery( + candidate: &Candidate, + run: &str, + device_rebind: Option, + host_boot_micros: Option, ) -> Result { let lock = state::Lock::acquire_existing(&root(candidate, run)?)?; let value = Self { - pin: Pin::read(candidate, run)?, + pin: Pin::read_with_rebind(candidate, run, device_rebind)?, _lock: lock, }; + if let Some(rebind) = device_rebind { + rebind.definitely_dead_before_boot( + &value.pin.record.process, + host_boot_micros.ok_or_else(refused)?, + )?; + } value.verify()?; Ok(value) } pub(in crate::provider::graph) fn verify(&self) -> Result<(), CandidateError> { + let pathname = + fs::symlink_metadata(self.pin.root.join("operation.lock")).map_err(|_| refused())?; + if !pathname.is_file() + || pathname.nlink() != 1 + || !private(&pathname) + || id(&pathname) != self._lock.identity()? + { + return Err(refused()); + } self.pin.verify_files()?; if identity::alive(self.pin.record.process.pid)? { return Err(refused()); diff --git a/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs b/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs index 5b21b6a10..b81f11ee3 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport_tests.rs @@ -201,6 +201,78 @@ fn interrupted_socket_move_resumes_but_replaced_owner_refuses() { fs::remove_dir_all(root).unwrap(); } +#[cfg(target_os = "macos")] +#[test] +fn selected_legacy_device_socket_move_resumes_only_exact_retirement() { + let (_fixture, candidate, run, _original_owner, root) = abandoned_publisher(); + let receipt = "f".repeat(64); + let current = id(&fs::symlink_metadata(&root).unwrap()).0; + let rebind = DeviceRebind { + old: current.checked_add(1).unwrap(), + current, + }; + let mut record: Record = state::read(&root.join("owner.json")).unwrap(); + assert_eq!(record.parent.0, current); + assert_eq!(record.socket.0, current); + record.parent.0 = rebind.old; + record.socket.0 = rebind.old; + let bytes = serde_json::to_vec(&record).unwrap(); + fs::write(root.join("owner.json"), &bytes).unwrap(); + let owner = format!("{:x}", Sha256::digest(&bytes)); + assert!(Pin::read(&candidate, &run).is_err()); + let lock = state::Lock::acquire_existing(&root).unwrap(); + let pin = Pin::read_with_rebind(&candidate, &run, Some(rebind)).unwrap(); + let intent = Retirement { + version: 1, + candidate: candidate.checkout.clone(), + run: run.clone(), + receipt_sha256: receipt.clone(), + owner_sha256: owner.clone(), + parent: pin.record.parent, + lock: lock.identity().unwrap(), + socket: pin.record.socket, + record: pin.record_id, + owner: pin.record, + }; + state::write(&retirement_path(&root, &owner), &intent).unwrap(); + fs::rename(root.join("control.sock"), retired_path(&root, &owner, true)).unwrap(); + drop(lock); + assert!(retire_recovered_publisher(&candidate, &run, &owner, &receipt).is_err()); + retire_recovered_publisher_recovery(&candidate, &run, &owner, &receipt, Some(rebind)).unwrap(); + let retired = Retired::acquire(&candidate, &run).unwrap().unwrap(); + retired + .verify_recovery_with_rebind(&candidate, &run, &owner, &receipt, Some(rebind)) + .unwrap(); + assert!( + retired + .verify_recovery(&candidate, &run, &owner, &receipt) + .is_err() + ); + drop(retired); + fs::remove_dir_all(root).unwrap(); +} + +#[cfg(target_os = "macos")] +#[test] +fn dead_owner_guard_refuses_replaced_foreground_lock_path() { + let (_fixture, candidate, run, _owner, root) = abandoned_publisher(); + let dead = DeadOwner::acquire(&candidate, &run).unwrap(); + dead.verify().unwrap(); + let pathname = root.join("operation.lock"); + let saved = root.join("held-operation.lock"); + fs::rename(&pathname, &saved).unwrap(); + fs::write(&pathname, b"replacement").unwrap(); + fs::set_permissions(&pathname, fs::Permissions::from_mode(0o600)).unwrap(); + let replacement = id(&fs::symlink_metadata(&pathname).unwrap()); + assert!(dead.verify().is_err()); + assert_eq!(id(&fs::symlink_metadata(&pathname).unwrap()), replacement); + assert_eq!(fs::read(&pathname).unwrap(), b"replacement"); + drop(dead); + fs::remove_file(&pathname).unwrap(); + fs::rename(&saved, &pathname).unwrap(); + fs::remove_dir_all(root).unwrap(); +} + #[cfg(target_os = "macos")] #[test] fn replaced_socket_or_inherited_listener_refuses_retirement() { diff --git a/packages/runtime-core/src/provider/graph/host_pin_recovery.rs b/packages/runtime-core/src/provider/graph/host_pin_recovery.rs new file mode 100644 index 000000000..a41c94b31 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/host_pin_recovery.rs @@ -0,0 +1,704 @@ +//! Explicit, selected legacy host-pin migration for one retained graph run. +//! +//! The private witness precedes dead-owner cleanup. It never edits historical graph, +//! publisher, control, or dependency receipts and grants no ordinary replay authority. +use super::{Candidate, CandidateError, Engine, Kind, Receipt, directory, foreground, load, state}; +use crate::provider::{ + ProjectShareIntent, + host_pin::DeviceRebind, + lifecycle, + relay_owner::publication::{LegacyControl, PinnedEndpoint}, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + fs::{self, OpenOptions}, + io::{Read, Seek, SeekFrom}, + os::unix::fs::{MetadataExt, OpenOptionsExt}, + path::{Path, PathBuf}, +}; + +use super::{dependency_slots::LegacyReservation, foreground::transport::LegacyPublisher}; + +const FILE: &str = "host-pin-recovery.json"; +const LIMIT: u64 = 64 * 1024; +const GUEST_IDENTITY_ACK: &str = "host-pin-guest-identity-v1\n"; + +fn refused() -> CandidateError { + CandidateError::new( + "graph_host_pin_recovery", + "Legacy host pin selection is incomplete or changed; receipts and data were preserved.", + ) +} +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +/// Guest::boot_id is sourced from the host Owner receipt. Execute a read-only +/// guest check so a changed receipt cannot impersonate the running kernel boot. +/// execute_cleanup itself compares /proc boot ID and /storage owner under the +/// held Engine/Guest lease before this fixed acknowledgement is emitted. +fn verify_guest_identity(engine: &Engine<'_>) -> Result<(), CandidateError> { + let acknowledged = engine + .guest() + .execute_cleanup("printf 'host-pin-guest-identity-v1\\n'", &[]) + .map_err(|_| refused())?; + if acknowledged != GUEST_IDENTITY_ACK { + return Err(refused()); + } + Ok(()) +} +fn read_raw(path: &Path, limit: u64) -> Result, CandidateError> { + read_raw_with(path, limit, || {}) +} +fn read_raw_with( + path: &Path, + limit: u64, + after_read: impl FnOnce(), +) -> Result, CandidateError> { + let parent = path.parent().ok_or_else(refused)?; + state::check_private_directory(parent)?; + let mut file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path) + .map_err(|_| refused())?; + let metadata = file.metadata().map_err(|_| refused())?; + // SAFETY: geteuid has no arguments or side effects. + if !metadata.is_file() + || metadata.nlink() != 1 + || metadata.mode() & 0o7777 != 0o600 + || metadata.uid() != unsafe { libc::geteuid() } + || metadata.len() == 0 + || metadata.len() > limit + { + return Err(refused()); + } + let mut bytes = Vec::new(); + file.by_ref() + .take(limit + 1) + .read_to_end(&mut bytes) + .map_err(|_| refused())?; + after_read(); + file.seek(SeekFrom::Start(0)).map_err(|_| refused())?; + let mut confirmation = Vec::new(); + file.by_ref() + .take(limit + 1) + .read_to_end(&mut confirmation) + .map_err(|_| refused())?; + let retained = file.metadata().map_err(|_| refused())?; + let pathname = fs::symlink_metadata(path).map_err(|_| refused())?; + if bytes != confirmation + || bytes.len() as u64 != metadata.len() + || [retained.dev(), pathname.dev()] != [metadata.dev(); 2] + || [retained.ino(), pathname.ino()] != [metadata.ino(); 2] + || [retained.len(), pathname.len()] != [metadata.len(); 2] + || [retained.uid(), pathname.uid()] != [metadata.uid(); 2] + || [retained.mode(), pathname.mode()] != [metadata.mode(); 2] + || [retained.nlink(), pathname.nlink()] != [metadata.nlink(); 2] + || [retained.mtime(), pathname.mtime()] != [metadata.mtime(); 2] + || [retained.mtime_nsec(), pathname.mtime_nsec()] != [metadata.mtime_nsec(); 2] + || [retained.ctime(), pathname.ctime()] != [metadata.ctime(); 2] + || [retained.ctime_nsec(), pathname.ctime_nsec()] != [metadata.ctime_nsec(); 2] + { + return Err(refused()); + } + Ok(bytes) +} +fn absent(path: &Path) -> Result { + match fs::symlink_metadata(path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true), + _ => Err(refused()), + } +} + +fn exact_lock_path(root: &Path, held: &state::Lock) -> Result<(), CandidateError> { + let pathname = fs::symlink_metadata(root.join("operation.lock")).map_err(|_| refused())?; + if !pathname.is_file() + || pathname.nlink() != 1 + || (pathname.dev(), pathname.ino()) != held.identity()? + { + return Err(refused()); + } + Ok(()) +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Witness { + version: u8, + candidate: PathBuf, + run: String, + owner: String, + namespace: String, + plan: String, + graph_sha256: String, + provider_sha256: String, + host_boot_micros: u64, + previous_guest_boot: String, + current_guest_boot: String, + rebind: DeviceRebind, + publisher: LegacyPublisher, + control_root: PathBuf, + control: LegacyControl, + reservation: Option, + source_shared: Option, + retained_volumes: BTreeMap, + qualification: String, +} + +impl Witness { + pub(super) fn rebind(&self) -> DeviceRebind { + self.rebind + } + pub(super) fn host_boot_micros(&self) -> u64 { + self.host_boot_micros + } + pub(super) fn publisher_sha256(&self) -> &str { + &self.publisher.owner_sha256 + } + pub(super) fn reservation(&self) -> Option<&LegacyReservation> { + self.reservation.as_ref() + } + pub(super) fn control(&self) -> &LegacyControl { + &self.control + } + pub(super) fn control_root(&self) -> &Path { + &self.control_root + } + pub(super) fn graph_sha256(&self) -> &str { + &self.graph_sha256 + } + pub(super) fn matches_graph(&self, receipt: &Receipt) -> bool { + self.run == receipt.run + && self.owner == receipt.owner + && self.namespace == receipt.namespace + && self.plan == receipt.plan_id + && self.source_shared == receipt.source.as_ref().and_then(|s| s.shared.clone()) + } +} + +pub(super) struct Guard { + witness: Witness, + witness_path: PathBuf, + control_root: PathBuf, + control_lock: state::Lock, +} + +pub(super) struct CleanupProof<'a> { + pub original: &'a Receipt, + pub sha256: &'a str, + pub current_is_original: bool, + pub allow_absent_reservation: bool, + pub publisher_may_be_partial: bool, +} +impl Guard { + pub(super) fn witness(&self) -> &Witness { + &self.witness + } + pub(super) fn verify_lock(&self) -> Result<(), CandidateError> { + let retained: Witness = + serde_json::from_slice(&read_raw(&self.witness_path, LIMIT)?).map_err(|_| refused())?; + if retained != self.witness { + return Err(refused()); + } + let context = + super::host_relay::context(&self.witness.owner, &self.witness.previous_guest_boot)?; + let control = PinnedEndpoint::load_legacy_recovery( + &self.witness.control_root, + context, + self.witness.rebind, + self.witness.host_boot_micros, + )?; + if control.legacy_summary() != self.witness.control { + return Err(refused()); + } + exact_lock_path(&self.control_root, &self.control_lock) + } +} + +fn path(candidate: &Candidate, run: &str) -> Result { + Ok(directory(candidate, run)?.join(FILE)) +} +pub(super) fn load_witness( + candidate: &Candidate, + run: &str, +) -> Result, CandidateError> { + let file = path(candidate, run)?; + absent(&file.with_extension("pending"))?; + if absent(&file).is_ok() { + return Ok(None); + } + let bytes = read_raw(&file, LIMIT)?; + let witness: Witness = serde_json::from_slice(&bytes).map_err(|_| refused())?; + if witness.version != 1 || witness.candidate != candidate.checkout || witness.run != run { + return Err(refused()); + } + Ok(Some(witness)) +} + +/// Only the exact old publisher may select the historical overlay. A successor +/// with current-device pins uses ordinary strict recovery despite retained history. +pub(super) fn selected_for_old_publisher( + candidate: &Candidate, + run: &str, +) -> Result, CandidateError> { + let Some(witness) = load_witness(candidate, run)? else { + return Ok(None); + }; + let recorded = foreground::transport::Pin::legacy_recorded_device(candidate, run)?; + if recorded == witness.rebind.current { + return Ok(None); + } + if recorded != witness.rebind.old { + return Err(refused()); + } + Ok(Some(witness)) +} + +fn verify_volume_projections( + engine: &Engine<'_>, + receipt: &Receipt, + selected: &BTreeMap, +) -> Result<(), CandidateError> { + let keys = receipt + .resources + .iter() + .filter(|(_, value)| value.kind == Kind::Volume) + .map(|(key, _)| key.clone()) + .collect::>(); + if keys.len() != selected.len() || keys.iter().any(|key| !selected.contains_key(key)) { + return Err(refused()); + } + for key in keys { + let resource = &receipt.resources[&key]; + let actual = super::inspect_resource(engine, receipt, resource)?.ok_or_else(refused)?; + let expected = selected.get(&key).ok_or_else(refused)?; + if expected["expected_name"] != resource.name + || expected["observed_name"] != actual["Name"] + || expected["observed_created_at"] != actual["CreatedAt"] + || expected["observed_mountpoint"] != actual["Mountpoint"] + || expected["observed_driver"] != actual["Driver"] + || expected["observed_labels_sha256"] + != digest(&serde_json::to_vec(&actual["Labels"]).map_err(|_| refused())?) + { + return Err(refused()); + } + let labels = expected["labels"].as_object().ok_or_else(refused)?; + if labels + .iter() + .any(|(name, value)| actual["Labels"].get(name) != Some(value)) + { + return Err(refused()); + } + } + Ok(()) +} + +/// Caller already holds the Engine lease and exact foreground owner lock. +/// The returned control lock remains held across graph cleanup effects. +pub(super) fn acquire_for_cleanup( + candidate: &Candidate, + run: &str, + engine: &Engine<'_>, + proof: CleanupProof<'_>, + witness: Witness, +) -> Result { + lifecycle::host_filesystem::no_auxiliary_update(candidate)?; + if witness.graph_sha256 != proof.sha256 + || !witness.matches_graph(proof.original) + || witness.candidate != candidate.checkout + || witness.run != run + || witness.qualification + != "explicit-legacy-device-rebind-original-volume-continuity-unproven" + || lifecycle::host_filesystem::host_boot_micros()? != witness.host_boot_micros + { + return Err(refused()); + } + witness + .rebind + .definitely_dead_before_boot(&witness.publisher.process, witness.host_boot_micros)?; + if proof.current_is_original + && digest(&read_raw( + &directory(candidate, run)?.join("state.json"), + 2 * 1024 * 1024, + )?) != witness.graph_sha256 + { + return Err(refused()); + } + let owner = state::Owner::load(candidate)?; + lifecycle::verify_disks(candidate, &owner)?; + verify_guest_identity(engine)?; + let bytes = read_raw( + &candidate.state_root.join("run/smolvm/owner.json"), + 1024 * 1024, + )?; + if digest(&bytes) != witness.provider_sha256 + || owner.token != witness.owner + || owner.guest_boot_id.as_deref() != Some(&witness.current_guest_boot) + || owner.previous_guest_boot_id.as_deref() != Some(&witness.previous_guest_boot) + || engine.guest().incarnation() != witness.owner + || engine.guest().boot_id() != witness.current_guest_boot + || owner + .storage + .as_ref() + .is_none_or(|disk| disk.device != witness.rebind.current) + || owner + .overlay + .as_ref() + .is_none_or(|disk| disk.device != witness.rebind.current) + { + return Err(refused()); + } + if !proof.publisher_may_be_partial { + let publisher = foreground::transport::Pin::legacy_summary( + candidate, + run, + witness.rebind, + witness.host_boot_micros, + )?; + if publisher != witness.publisher { + return Err(refused()); + } + } + let control_root = witness.control_root.join("relay-control"); + let control_lock = state::Lock::acquire_existing(&control_root)?; + absent(&foreground::transport::root(candidate, run)?.join("owner.pending"))?; + absent(&control_root.join("owner.pending"))?; + let context = super::host_relay::context(&witness.owner, &witness.previous_guest_boot)?; + let control = PinnedEndpoint::load_legacy_recovery( + &witness.control_root, + context, + witness.rebind, + witness.host_boot_micros, + )? + .legacy_summary(); + if control != witness.control { + return Err(refused()); + } + if let Some(selected) = &witness.reservation { + super::dependency_slots::verify_legacy_remaining( + candidate, + run, + witness.rebind, + selected, + proof.allow_absent_reservation, + )?; + } else if super::dependency_slots::inspect_legacy( + candidate, + run, + witness.rebind, + witness.host_boot_micros, + )? + .is_some() + { + return Err(refused()); + } + verify_volume_projections(engine, proof.original, &witness.retained_volumes)?; + let guard = Guard { + witness_path: path(candidate, run)?, + witness, + control_root, + control_lock, + }; + guard.verify_lock()?; + Ok(guard) +} + +struct Selected { + witness: Witness, + foreground_root: PathBuf, + foreground_lock: state::Lock, + control_lock_root: PathBuf, + control_lock: state::Lock, +} + +impl Selected { + fn verify( + &self, + candidate: &Candidate, + run: &str, + engine: &Engine<'_>, + ) -> Result<(), CandidateError> { + for (root, lock) in [ + (&self.foreground_root, &self.foreground_lock), + (&self.control_lock_root, &self.control_lock), + ] { + exact_lock_path(root, lock)?; + } + if self.foreground_root != foreground::transport::root(candidate, run)? + || self.control_lock_root != self.witness.control_root.join("relay-control") + || self.witness != select_content(candidate, run, engine)? + { + return Err(refused()); + } + Ok(()) + } +} + +fn select( + candidate: &Candidate, + run: &str, + engine: &Engine<'_>, +) -> Result { + let (receipt, _) = load(candidate, engine, run)?; + let startup = receipt.relay_startup.as_ref().ok_or_else(refused)?; + let foreground_root = foreground::transport::root(candidate, run)?; + let foreground_lock = state::Lock::acquire_existing(&foreground_root)?; + let control_lock_root = startup.control_root.join("relay-control"); + let control_lock = state::Lock::acquire_existing(&control_lock_root)?; + let selected = Selected { + witness: select_content(candidate, run, engine)?, + foreground_root, + foreground_lock, + control_lock_root, + control_lock, + }; + selected.verify(candidate, run, engine)?; + Ok(selected) +} + +fn select_content( + candidate: &Candidate, + run: &str, + engine: &Engine<'_>, +) -> Result { + lifecycle::host_filesystem::no_auxiliary_update(candidate)?; + let owner = state::Owner::load(candidate)?; + lifecycle::verify_disks(candidate, &owner)?; + verify_guest_identity(engine)?; + let host_boot_micros = lifecycle::host_filesystem::host_boot_micros()?; + let current_guest_boot = owner.guest_boot_id.clone().ok_or_else(refused)?; + let previous_guest_boot = owner.previous_guest_boot_id.clone().ok_or_else(refused)?; + if current_guest_boot == previous_guest_boot + || current_guest_boot != engine.guest().boot_id() + || owner.token != engine.guest().incarnation() + { + return Err(refused()); + } + let (receipt, root) = load(candidate, engine, run)?; + for name in [ + "state.pending", + "dead-owner-cleanup.pending", + "relay-cleanup-bridges.pending", + ] { + absent(&root.join(name))?; + } + if receipt.phase != "ready-observed" || receipt.relay_cleanup.is_some() { + return Err(refused()); + } + let startup = receipt.relay_startup.as_ref().ok_or_else(refused)?; + let graph_sha256 = digest(&read_raw(&root.join("state.json"), 2 * 1024 * 1024)?); + let provider_sha256 = digest(&read_raw( + &candidate.state_root.join("run/smolvm/owner.json"), + 1024 * 1024, + )?); + let publisher_root = foreground::transport::root(candidate, run)?; + absent(&publisher_root.join("owner.pending"))?; + absent(&startup.control_root.join("relay-control/owner.pending"))?; + let current_device = fs::symlink_metadata(&publisher_root) + .map_err(|_| refused())? + .dev(); + let rebind = DeviceRebind { + old: foreground::transport::Pin::legacy_recorded_device(candidate, run)?, + current: current_device, + }; + if rebind.old == rebind.current { + return Err(refused()); + } + if owner + .storage + .as_ref() + .is_none_or(|disk| disk.device != rebind.current) + || owner + .overlay + .as_ref() + .is_none_or(|disk| disk.device != rebind.current) + { + return Err(refused()); + } + let publisher = + foreground::transport::Pin::legacy_summary(candidate, run, rebind, host_boot_micros)?; + let context = super::host_relay::context(&receipt.owner, &previous_guest_boot)?; + let control = PinnedEndpoint::load_legacy_recovery( + &startup.control_root, + context, + rebind, + host_boot_micros, + )? + .legacy_summary(); + let reservation = + super::dependency_slots::inspect_legacy(candidate, run, rebind, host_boot_micros)?; + if receipt + .source + .as_ref() + .and_then(|source| source.shared.as_ref()) + .is_some_and(|shared| shared.device != rebind.old) + || owner.project_share.as_ref() + != receipt + .source + .as_ref() + .and_then(|s| s.shared.as_ref()) + .map(|shared| { + // Historical shared source is pinned to the old host device. Only the + // selected device may differ from the currently approved project share. + let mut expected = shared.clone(); + expected.device = rebind.current; + expected + }) + .as_ref() + { + return Err(refused()); + } + if let Some(share) = &owner.project_share { + share.validate()?; + } + let mut retained_volumes = BTreeMap::new(); + for (key, resource) in receipt + .resources + .iter() + .filter(|(_, value)| value.kind == Kind::Volume) + { + let actual = super::inspect_resource(engine, &receipt, resource)?.ok_or_else(refused)?; + let labels = super::expected_labels(&receipt, resource) + .as_object() + .ok_or_else(refused)? + .keys() + .map(|label| (label.clone(), actual["Labels"][label].clone())) + .collect::>(); + retained_volumes.insert( + key.clone(), + json!({"expected_name":resource.name,"observed_name":actual["Name"], + "observed_created_at":actual["CreatedAt"], + "observed_mountpoint":actual["Mountpoint"], + "observed_driver":actual["Driver"], + "observed_labels_sha256":digest(&serde_json::to_vec(&actual["Labels"]).map_err(|_| refused())?), + "labels":labels}), + ); + } + Ok(Witness { + version: 1, + candidate: candidate.checkout.clone(), + run: run.into(), + owner: receipt.owner, + namespace: receipt.namespace, + plan: receipt.plan_id, + graph_sha256, + provider_sha256, + host_boot_micros, + previous_guest_boot, + current_guest_boot, + rebind, + publisher, + control_root: startup.control_root.clone(), + control, + reservation, + source_shared: receipt.source.and_then(|s| s.shared), + retained_volumes, + qualification: "explicit-legacy-device-rebind-original-volume-continuity-unproven".into(), + }) +} + +fn selected_sha256(witness: &Witness) -> Result { + Ok(digest(&serde_json::to_vec(witness).map_err(|_| refused())?)) +} + +/// Read-only exact selection. Existing completed recovery history is retained. +pub fn inspect(candidate: &Candidate, run: &str) -> Result { + let engine = Engine::connect_cleanup_wait(candidate)?; + let selected = select(candidate, run, &engine)?; + Ok( + json!({"run":run,"selection_sha256":selected_sha256(&selected.witness)?,"old_device":selected.witness.rebind.old,"new_device":selected.witness.rebind.current,"qualification":selected.witness.qualification}), + ) +} + +/// Explicitly publish one private immutable witness before cleanup can consume an overlay. +pub fn recover(candidate: &Candidate, run: &str, expected: &str) -> Result { + if expected.len() != 64 || !expected.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(refused()); + } + let engine = Engine::connect_cleanup_wait(candidate)?; + let selected = select(candidate, run, &engine)?; + if selected_sha256(&selected.witness)? != expected { + return Err(refused()); + } + if let Some(existing) = load_witness(candidate, run)? { + if existing == selected.witness { + return Ok( + json!({"run":run,"selection_sha256":expected,"witness_published":true,"already_published":true,"qualification":selected.witness.qualification}), + ); + } + return Err(refused()); + } + selected.verify(candidate, run, &engine)?; + state::write(&path(candidate, run)?, &selected.witness)?; + Ok( + json!({"run":run,"selection_sha256":expected,"witness_published":true,"qualification":selected.witness.qualification}), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{io::Write, os::unix::fs::PermissionsExt}; + + #[test] + fn raw_selection_refuses_same_inode_in_place_substitution() { + let fixture = super::super::tests::Fixture::new(); + let path = fixture.0.join("selected.json"); + fs::write(&path, b"first").unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + let before = fs::symlink_metadata(&path).unwrap(); + let result = read_raw_with(&path, 64, || { + let mut file = OpenOptions::new().write(true).open(&path).unwrap(); + file.write_all(b"other").unwrap(); + file.sync_all().unwrap(); + }); + assert!(result.is_err()); + assert_eq!(fs::symlink_metadata(&path).unwrap().ino(), before.ino()); + } + + #[test] + fn device_overlay_requires_only_one_number_change_and_exact_inode() { + let translated = DeviceRebind { + old: 10, + current: 20, + }; + assert!(translated.matches((10, 77), (20, 77))); + for (old, current) in [ + ((20, 77), (20, 77)), + ((10, 77), (10, 77)), + ((10, 77), (20, 78)), + ((10, 0), (20, 0)), + ] { + assert!(!translated.matches(old, current)); + } + } + + #[test] + fn held_lock_path_replacement_refuses_and_preserves_foreign_replacement() { + let fixture = super::super::tests::Fixture::new(); + let lock = state::Lock::acquire(&fixture.0).unwrap(); + exact_lock_path(&fixture.0, &lock).unwrap(); + let pathname = fixture.0.join("operation.lock"); + let saved = fixture.0.join("held-operation.lock"); + fs::rename(&pathname, &saved).unwrap(); + fs::write(&pathname, b"foreign replacement").unwrap(); + fs::set_permissions(&pathname, fs::Permissions::from_mode(0o600)).unwrap(); + let replacement = fs::symlink_metadata(&pathname).unwrap(); + assert!(exact_lock_path(&fixture.0, &lock).is_err()); + let after = fs::symlink_metadata(&pathname).unwrap(); + assert_eq!( + (after.dev(), after.ino()), + (replacement.dev(), replacement.ino()) + ); + assert_eq!(fs::read(&pathname).unwrap(), b"foreign replacement"); + drop(lock); + // The fixture is private and isolated; restore the original pathname + // before its directory is torn down. + fs::remove_file(&pathname).unwrap(); + fs::rename(&saved, &pathname).unwrap(); + } +} diff --git a/packages/runtime-core/src/provider/graph/host_relay.rs b/packages/runtime-core/src/provider/graph/host_relay.rs index 94a5145d8..49946e7f5 100644 --- a/packages/runtime-core/src/provider/graph/host_relay.rs +++ b/packages/runtime-core/src/provider/graph/host_relay.rs @@ -489,6 +489,25 @@ pub(super) fn inspect_cleanup( remove_data: bool, environment: &super::super::environment_recovery::GraphInventory, bridge_selection: &bridges::cleanup::Selection, +) -> Result<[u8; 32], CandidateError> { + inspect_cleanup_recovery( + candidate, + engine, + expected, + remove_data, + environment, + bridge_selection, + None, + ) +} +pub(super) fn inspect_cleanup_recovery( + candidate: &Candidate, + engine: &Engine<'_>, + expected: &Receipt, + remove_data: bool, + environment: &super::super::environment_recovery::GraphInventory, + bridge_selection: &bridges::cleanup::Selection, + host_pin: Option<&super::host_pin_recovery::Witness>, ) -> Result<[u8; 32], CandidateError> { let (receipt, root) = archive::load_confirmation(candidate, engine, &expected.run)?; match fs::symlink_metadata(root.join("state.pending")) { @@ -533,10 +552,10 @@ pub(super) fn inspect_cleanup( require_retained_volume(resource, value.is_some(), remove_data)?; observations.insert(key, value); } - if &bridges::cleanup::read(engine, &receipt, &root)? != bridge_selection { + if &bridges::cleanup::read_recovery(engine, &receipt, &root, host_pin)? != bridge_selection { return Err(refused()); } - bridges::cleanup::verify(candidate, engine, &receipt, bridge_selection)?; + bridges::cleanup::verify_recovery(candidate, engine, &receipt, bridge_selection, host_pin)?; environment::verify_cleanup(candidate, engine, &receipt, &root, environment)?; probes::verify_cleanup(engine, &receipt)?; startup::verify_cleanup(engine, &receipt)?; diff --git a/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs b/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs index 33f1aa01e..d8b89dc68 100644 --- a/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs +++ b/packages/runtime-core/src/provider/graph/live_owner_cleanup.rs @@ -380,7 +380,7 @@ fn finish_retirement( &intent.foreground_sha256, &digest(receipt)?, )?; - dependency_slots::recover_cleaned(candidate, receipt)?; + dependency_slots::recover_cleaned(candidate, receipt, None)?; engine.guest().verify() } diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index 0236bce3c..d43122f1e 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -18,9 +18,13 @@ mod dependency_hosts; #[cfg(target_os = "macos")] mod dependency_slots; #[cfg(target_os = "macos")] +mod host_pin_recovery; +#[cfg(target_os = "macos")] pub use dependency_slots::{ inspect as dependency_reservations, recover_orphan as recover_dependency_reservation, }; +#[cfg(target_os = "macos")] +pub use host_pin_recovery::{inspect as inspect_host_pin_recovery, recover as recover_host_pins}; mod initializer_cache; mod volume_subpaths; pub use dependency_hosts::dependency_address; diff --git a/packages/runtime-core/src/provider/host_pin.rs b/packages/runtime-core/src/provider/host_pin.rs new file mode 100644 index 000000000..f583198d4 --- /dev/null +++ b/packages/runtime-core/src/provider/host_pin.rs @@ -0,0 +1,40 @@ +//! Exact device-number translation for explicitly selected pre-reboot host pins. +//! This is never used by ordinary ownership, source replay, or guest observations. +use super::identity::{self, ProcessIdentity}; +use crate::CandidateError; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct DeviceRebind { + pub old: u64, + pub current: u64, +} + +impl DeviceRebind { + pub fn matches(self, recorded: (u64, u64), observed: (u64, u64)) -> bool { + self.old != self.current + && recorded.0 == self.old + && observed.0 == self.current + && recorded.1 != 0 + && recorded.1 == observed.1 + } + + pub fn definitely_dead_before_boot( + self, + process: &ProcessIdentity, + host_boot_micros: u64, + ) -> Result<(), CandidateError> { + if host_boot_micros == 0 + || process.start_micros == 0 + || process.start_micros >= host_boot_micros + || identity::alive(process.pid)? + { + return Err(CandidateError::new( + "host_pin_recovery", + "Legacy host pin owner is not proved dead before this host boot.", + )); + } + Ok(()) + } +} diff --git a/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs b/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs index 6c9e10634..2b7ad0ddb 100644 --- a/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs +++ b/packages/runtime-core/src/provider/lifecycle/host_filesystem.rs @@ -35,7 +35,7 @@ fn refused(detail: &str) -> CandidateError { } #[cfg(target_os = "macos")] -fn host_boot_micros() -> Result { +pub(crate) fn host_boot_micros() -> Result { let mut time = std::mem::MaybeUninit::::zeroed(); let mut length = std::mem::size_of::(); // SAFETY: read-only sysctl writes an exactly sized timeval; no input or retained pointers. @@ -74,7 +74,7 @@ fn host_boot_micros() -> Result { )) } -fn no_auxiliary_update(candidate: &Candidate) -> Result<(), CandidateError> { +pub(crate) fn no_auxiliary_update(candidate: &Candidate) -> Result<(), CandidateError> { crate::provider::network_update::require_complete(candidate)?; for name in [ "owner.pending", diff --git a/packages/runtime-core/src/provider/mod.rs b/packages/runtime-core/src/provider/mod.rs index e851f1890..d1203b751 100644 --- a/packages/runtime-core/src/provider/mod.rs +++ b/packages/runtime-core/src/provider/mod.rs @@ -19,6 +19,8 @@ pub mod environment; mod environment_probe_test; pub mod environment_recovery; pub mod graph; +#[cfg(target_os = "macos")] +mod host_pin; pub use engine::{EngineInfo, info as engine_info}; #[cfg(all(test, target_os = "macos", target_arch = "aarch64"))] mod gateway_probe_test; diff --git a/packages/runtime-core/src/provider/relay_owner/publication.rs b/packages/runtime-core/src/provider/relay_owner/publication.rs index 2210b12ec..178e0a97d 100644 --- a/packages/runtime-core/src/provider/relay_owner/publication.rs +++ b/packages/runtime-core/src/provider/relay_owner/publication.rs @@ -7,6 +7,7 @@ use super::{ use crate::{ CandidateError, provider::{ + host_pin::DeviceRebind, identity::{self, ProcessIdentity}, state, }, @@ -102,6 +103,16 @@ pub struct PinnedEndpoint { receipt: Receipt, receipt_id: FileId, bytes: Vec, + device_rebind: Option, +} +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct LegacyControl { + pub receipt_sha256: String, + pub receipt_id: FileId, + pub parent: FileId, + pub endpoint: FileId, + pub process: ProcessIdentity, } impl PinnedEndpoint { pub(crate) fn runtime_root(&self) -> &Path { @@ -134,6 +145,13 @@ impl PinnedEndpoint { Self::read(Paths::new(root)?, context) } fn read(paths: Paths, context: Context) -> Result { + Self::read_with_rebind(paths, context, None) + } + fn read_with_rebind( + paths: Paths, + context: Context, + device_rebind: Option, + ) -> Result { let parent = paths.parent()?; let mut file = OpenOptions::new() .read(true) @@ -160,7 +178,9 @@ impl PinnedEndpoint { || context.boot == [0; 16] || receipt.owner == [0; 16] || receipt.socket != paths.socket - || receipt.parent != parent + || !device_rebind.map_or(receipt.parent == parent, |v| { + v.matches(receipt.parent, parent) + }) || receipt.endpoint.1 == 0 || !receipt.process.executable.is_absolute() { @@ -179,8 +199,30 @@ impl PinnedEndpoint { receipt, receipt_id: id(&m), bytes, + device_rebind, }) } + pub(crate) fn load_legacy_recovery( + root: &Path, + context: Context, + device_rebind: DeviceRebind, + host_boot_micros: u64, + ) -> Result { + let pin = Self::read_with_rebind(Paths::new(root)?, context, Some(device_rebind))?; + pin.verify_dead()?; + device_rebind.definitely_dead_before_boot(&pin.receipt.process, host_boot_micros)?; + dead::no_listener(&pin.paths.socket)?; + Ok(pin) + } + pub(crate) fn legacy_summary(&self) -> LegacyControl { + LegacyControl { + receipt_sha256: format!("{:x}", Sha256::digest(&self.bytes)), + receipt_id: self.receipt_id, + parent: self.receipt.parent, + endpoint: self.receipt.endpoint, + process: self.receipt.process.clone(), + } + } /// Read-only predecessor proof: exact private receipt/socket, absent owner. #[cfg(target_os = "macos")] pub(crate) fn verify_dead(&self) -> Result<(), CandidateError> { @@ -198,12 +240,13 @@ impl PinnedEndpoint { Sha256::digest(&self.bytes).into() } fn verify_receipt(&self) -> Result<(), CandidateError> { - let current = Self::read( + let current = Self::read_with_rebind( self.paths.clone(), Context { runtime: self.receipt.runtime, boot: self.receipt.boot, }, + self.device_rebind, )?; if current.receipt_id != self.receipt_id || current.bytes != self.bytes { return Err(refused()); @@ -211,14 +254,24 @@ impl PinnedEndpoint { Ok(()) } fn verify_socket(&self) -> Result<(), CandidateError> { - if self.paths.parent()? != self.receipt.parent { + let parent = self.paths.parent()?; + if !self + .device_rebind + .map_or(parent == self.receipt.parent, |v| { + v.matches(self.receipt.parent, parent) + }) + { return Err(refused()); } let m = fs::symlink_metadata(&self.paths.socket).map_err(|_| refused())?; if !m.file_type().is_socket() || !private(&m) || m.nlink() != 1 - || id(&m) != self.receipt.endpoint + || !self + .device_rebind + .map_or(id(&m) == self.receipt.endpoint, |v| { + v.matches(self.receipt.endpoint, id(&m)) + }) { return Err(refused()); } @@ -353,6 +406,7 @@ impl ControlListener { receipt, receipt_id, bytes, + device_rebind: None, }; endpoint.verify_receipt()?; endpoint.verify_socket()?; diff --git a/packages/runtime-core/src/provider/relay_owner/publication/dead.rs b/packages/runtime-core/src/provider/relay_owner/publication/dead.rs index edec289f5..4c5390feb 100644 --- a/packages/runtime-core/src/provider/relay_owner/publication/dead.rs +++ b/packages/runtime-core/src/provider/relay_owner/publication/dead.rs @@ -41,6 +41,7 @@ impl Selection { receipt, receipt_id: self.record_id, bytes: self.bytes.clone(), + device_rebind: None, }) } } From 2404e436492213c104706d7589ff86735f6c2505 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 02:44:15 -0400 Subject: [PATCH 5/9] test(runtime): model absent publication recovery safety --- scripts/lib/tla-runtime-models.ts | 55 +++++++ tests/models/tla/README.md | 41 ++++++ .../absent-publication-recovery/Absent.tla | 136 ++++++++++++++++++ .../absent-publication-recovery/negative.cfg | 12 ++ .../absent-publication-recovery/positive.cfg | 12 ++ .../stale-selection.cfg | 12 ++ .../unconfirmed-retirement.cfg | 12 ++ .../unwitnessed-cleanup.cfg | 12 ++ 8 files changed, 292 insertions(+) create mode 100644 tests/models/tla/absent-publication-recovery/Absent.tla create mode 100644 tests/models/tla/absent-publication-recovery/negative.cfg create mode 100644 tests/models/tla/absent-publication-recovery/positive.cfg create mode 100644 tests/models/tla/absent-publication-recovery/stale-selection.cfg create mode 100644 tests/models/tla/absent-publication-recovery/unconfirmed-retirement.cfg create mode 100644 tests/models/tla/absent-publication-recovery/unwitnessed-cleanup.cfg diff --git a/scripts/lib/tla-runtime-models.ts b/scripts/lib/tla-runtime-models.ts index c0222485e..f2fd5799c 100644 --- a/scripts/lib/tla-runtime-models.ts +++ b/scripts/lib/tla-runtime-models.ts @@ -26,6 +26,61 @@ type ModelContract = { // Bounds and witnesses are reviewed contracts, not learned from each run. const contracts: readonly ModelContract[] = [ + { + name: "absent-publication-recovery", + module: "Absent", + states: 247, + invariant: "NoPrematurePublication", + action: "Publish", + fields: [ + "intent = TRUE", + "complete = FALSE", + "retired = FALSE", + "published = TRUE", + "unsafePublication = TRUE", + ], + additionalControls: [ + { + name: "unwitnessed-cleanup", + negative: true, + invariant: "NoUnwitnessedCleanup", + action: "CleanupOne", + fields: [ + "intent = FALSE", + "progress = 1", + 'engine = "recovery"', + 'foreground = "recovery"', + "unsafeCleanup = TRUE", + ], + }, + { + name: "stale-selection", + negative: true, + invariant: "NoUnwitnessedCleanup", + action: "CleanupOne", + fields: [ + "intent = TRUE", + "selected = 1", + "version = 2", + "progress = 1", + "unsafeCleanup = TRUE", + ], + }, + { + name: "unconfirmed-retirement", + negative: true, + invariant: "NoUnconfirmedRetirement", + action: "RetireAbsentPublisher", + fields: [ + "intent = FALSE", + "complete = FALSE", + "progress = 0", + "retired = TRUE", + "unsafeRetirement = TRUE", + ], + }, + ], + }, { name: "shared-https-lifetime", module: "SharedHttps", diff --git a/tests/models/tla/README.md b/tests/models/tla/README.md index cd08ad461..358a5d65a 100644 --- a/tests/models/tla/README.md +++ b/tests/models/tla/README.md @@ -16,6 +16,47 @@ pinned artifact. Each check has a 120-second timeout, 512 MiB Java heap, two wor and bounded output; temporary TLC metadata is removed after success or failure. No credentials or running VM are needed. +## Missing post-reboot publications + +`absent-publication-recovery/Absent.tla` checks one explicitly selected cleanup +racing one foreground publisher. The positive configuration explores 247 distinct +states, including one recovery-process crash, two non-volume cleanup effects and +one input-version change. Both participants acquire the foreground lock before the +Engine lease. Ordinary publication occurs under the foreground lock before Engine +admission, matching the implementation. The durable absence intent survives a +crash; a restart cannot publish until cleanup and separate absence retirement are +confirmed. A changed selection +cannot resume cleanup. Completion and retirement are separate steps, so a crash +between them remains visible. + +Four guard-removal controls require TLC exit 12 and a named same-state witness: + +| Control | Required failure | +| --- | --- | +| `negative` | `NoPrematurePublication` in `Publish`, with a durable intent, incomplete cleanup, no retirement and a newly published owner | +| `unwitnessed-cleanup` | `NoUnwitnessedCleanup` in `CleanupOne`, with both locks held but no intent | +| `stale-selection` | `NoUnwitnessedCleanup` in `CleanupOne`, with a durable intent but selected version 1 and current version 2 | +| `unconfirmed-retirement` | `NoUnconfirmedRetirement` in `RetireAbsentPublisher`, before cleanup or intent | + +| Model action | Implementation boundary under `packages/runtime-core/src/provider/` | +| --- | --- | +| `AcquireRecoveryForeground` / `AcquireRecoveryEngine` | `graph/absent_publication_cleanup.rs`: deterministic lock-only reservation before the Engine lease; matches foreground startup's lock order | +| `WriteIntent` | Private durable run-scoped absence intent before any cleanup effects | +| `CleanupOne` / `CommitCleanup` | Exact selected, data-retaining `cleanup_owned(..., false)` and stopped receipt confirmation; tagged absent bridge authority remains distinct from pinned predecessors | +| `RetireAbsentPublisher` | Separate durable absent-publication retirement proof tied to completed cleanup | +| `AcquirePublisherForeground` / `AcquirePublisherEngine` / `Publish` | `graph/foreground/transport.rs` admission barrier plus `graph/foreground.rs` Engine acquisition; explicit retired binding needs completed proof | +| `CrashRecovery` / `ChangeInputs` | Retained intent and exact retry; reinspection refuses changed host/guest boot, input bytes or resources | + +The model assumes validated legacy inputs, absent roots and resource observations +are summarized by one version. It represents cooperating writers under two kernel +locks and durable writes as atomic commits. It does not prove filesystem fsync, +FD/path identity, raw hashes, PID reuse, private permissions, actual guest or +physical host reboot, bridge cleanup, volume preservation, source-device migration, +later restore generations, browser readiness or performance. Concrete refusal, +interruption and data-marker tests remain required. The recorded old Owner is +legacy corroboration, never a reconstructed foreground Pin or proof of original +physical-volume continuity. No fairness or eventual recovery claim is made. + ## Shared HTTPS lifetime `shared-https-lifetime/SharedHttps.tla` checks the last-lease release racing diff --git a/tests/models/tla/absent-publication-recovery/Absent.tla b/tests/models/tla/absent-publication-recovery/Absent.tla new file mode 100644 index 000000000..a736a3182 --- /dev/null +++ b/tests/models/tla/absent-publication-recovery/Absent.tla @@ -0,0 +1,136 @@ +----------------------------- MODULE Absent ----------------------------- +EXTENDS Naturals, TLC +CONSTANTS EnforceIntent, EnforceBarrier, EnforceSelection, EnforceCompletion +VARIABLES foreground, engine, recovery, publisher, intent, progress, + complete, retired, published, version, selected, crashed, + unsafeCleanup, unsafePublication, unsafeRetirement +durable == <> +observations == <> +effects == <> +vars == <> +LocksHeld == foreground = "recovery" /\ engine = "recovery" +SelectionMatches == selected = version +\* Exact legacy inputs and resource/guest observations are one abstract version. +\* No old Pin is synthesized. The selection starts without any live publication. +Init == /\ foreground = "none" /\ engine = "none" + /\ recovery = "idle" /\ publisher = "idle" + /\ intent = FALSE /\ progress = 0 /\ complete = FALSE + /\ retired = FALSE /\ published = FALSE + /\ version = 1 /\ selected = 0 /\ crashed = FALSE + /\ unsafeCleanup = FALSE /\ unsafePublication = FALSE + /\ unsafeRetirement = FALSE +AcquireRecoveryForeground == + /\ recovery \in {"idle", "crashed"} /\ foreground = "none" + /\ foreground' = "recovery" /\ recovery' = "foreground" + /\ selected' = IF intent THEN selected ELSE version + /\ UNCHANGED <> +AcquireRecoveryEngine == + /\ recovery = "foreground" /\ foreground = "recovery" /\ engine = "none" + /\ engine' = "recovery" + /\ recovery' = IF retired THEN "retired" + ELSE IF complete THEN "complete" + ELSE IF intent THEN "cleaning" ELSE "leased" + /\ UNCHANGED <> +WriteIntent == + /\ recovery = "leased" /\ LocksHeld /\ SelectionMatches /\ ~published + /\ intent' = TRUE /\ recovery' = "cleaning" + /\ UNCHANGED <> +\* One effect represents one exact non-volume resource cleanup. Commit is separate. +CleanupOne == + /\ recovery \in {"leased", "cleaning"} /\ LocksHeld /\ ~published + /\ progress < 2 /\ (EnforceIntent => intent) + /\ (EnforceSelection => SelectionMatches) + /\ progress' = progress + 1 /\ recovery' = "cleaning" + /\ unsafeCleanup' = (unsafeCleanup \/ ~intent \/ ~SelectionMatches) + /\ UNCHANGED <> +CommitCleanup == + /\ recovery = "cleaning" /\ LocksHeld /\ intent + /\ SelectionMatches /\ progress = 2 /\ ~published + /\ complete' = TRUE /\ recovery' = "complete" + /\ UNCHANGED <> +RetireAbsentPublisher == + /\ recovery \in {"leased", "cleaning", "complete"} /\ LocksHeld + /\ SelectionMatches /\ ~published + /\ (EnforceCompletion => (intent /\ complete /\ progress = 2)) + /\ retired' = TRUE /\ recovery' = "retired" + /\ unsafeRetirement' = (unsafeRetirement \/ ~intent \/ ~complete \/ progress # 2) + /\ UNCHANGED <> +ReleaseRecovery == + /\ recovery = "retired" /\ LocksHeld + /\ foreground' = "none" /\ engine' = "none" /\ recovery' = "done" + /\ UNCHANGED <> +\* A crash releases kernel locks while retaining every committed durable field. +CrashRecovery == + /\ recovery \in {"foreground", "leased", "cleaning", "complete", "retired"} + /\ ~crashed /\ recovery' = "crashed" /\ crashed' = TRUE + /\ foreground' = "none" /\ engine' = "none" + /\ UNCHANGED <> +\* A cooperating identity/guest-boot change cannot occur under the Engine lease. +ChangeInputs == + /\ engine = "none" /\ version = 1 /\ version' = 2 + /\ UNCHANGED <> +RefuseRecovery == + /\ recovery \in {"leased", "cleaning", "complete"} /\ LocksHeld + /\ (~SelectionMatches \/ published) + /\ recovery' = "refused" /\ foreground' = "none" /\ engine' = "none" + /\ UNCHANGED <> +AcquirePublisherForeground == + /\ publisher = "idle" /\ foreground = "none" + /\ publisher' = "foreground" /\ foreground' = "publisher" + /\ UNCHANGED <> +AcquirePublisherEngine == + /\ publisher = "published" /\ foreground = "publisher" /\ engine = "none" + /\ publisher' = "active" /\ engine' = "publisher" + /\ UNCHANGED <> +ReleasePublisherEngine == + /\ publisher = "active" /\ engine = "publisher" + /\ publisher' = "running" /\ engine' = "none" + /\ UNCHANGED <> +FreshAllowed == ~intent /\ ~published +RetiredAllowed == intent /\ complete /\ retired /\ ~published +\* Ordinary bind publishes while holding the foreground lock, BEFORE Engine admission. +\* Recovery also needs that foreground lock, so intent and publication cannot race. +Publish == + /\ publisher = "foreground" /\ foreground = "publisher" /\ engine = "none" + /\ (FreshAllowed \/ RetiredAllowed \/ (~EnforceBarrier /\ ~published)) + /\ published' = TRUE /\ publisher' = "published" + /\ unsafePublication' = (unsafePublication \/ (intent /\ ~retired)) + /\ UNCHANGED <> +RefusePublisher == + /\ publisher = "foreground" /\ ~FreshAllowed /\ ~RetiredAllowed + /\ publisher' = "refused" /\ foreground' = "none" + /\ UNCHANGED <> +PublisherExit == + /\ publisher \in {"published", "active", "running"} /\ foreground = "publisher" + /\ publisher' = "exited" /\ foreground' = "none" /\ engine' = "none" + /\ UNCHANGED <> +Next == AcquireRecoveryForeground \/ AcquireRecoveryEngine \/ WriteIntent \/ CleanupOne + \/ CommitCleanup \/ RetireAbsentPublisher \/ ReleaseRecovery \/ CrashRecovery + \/ ChangeInputs \/ RefuseRecovery \/ AcquirePublisherForeground + \/ AcquirePublisherEngine \/ ReleasePublisherEngine \/ Publish \/ RefusePublisher + \/ PublisherExit +Spec == Init /\ [][Next]_vars +TypeOK == /\ foreground \in {"none", "recovery", "publisher"} + /\ engine \in {"none", "recovery", "publisher"} + /\ recovery \in {"idle", "foreground", "leased", "cleaning", "complete", + "retired", "crashed", "done", "refused"} + /\ publisher \in {"idle", "foreground", "published", "active", "running", + "refused", "exited"} + /\ intent \in BOOLEAN /\ progress \in 0..2 /\ complete \in BOOLEAN + /\ retired \in BOOLEAN /\ published \in BOOLEAN + /\ version \in 1..2 /\ selected \in 0..2 /\ crashed \in BOOLEAN + /\ unsafeCleanup \in BOOLEAN /\ unsafePublication \in BOOLEAN + /\ unsafeRetirement \in BOOLEAN +ForegroundBeforeEngine == engine # "none" => engine = foreground +NoUnwitnessedCleanup == ~unsafeCleanup +NoPrematurePublication == ~unsafePublication +NoUnconfirmedRetirement == ~unsafeRetirement +CompletionRequiresCleanup == complete => (intent /\ progress = 2) +======================================================================= diff --git a/tests/models/tla/absent-publication-recovery/negative.cfg b/tests/models/tla/absent-publication-recovery/negative.cfg new file mode 100644 index 000000000..16ebac56b --- /dev/null +++ b/tests/models/tla/absent-publication-recovery/negative.cfg @@ -0,0 +1,12 @@ +CONSTANTS EnforceIntent = TRUE + EnforceBarrier = FALSE + EnforceSelection = TRUE + EnforceCompletion = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK + ForegroundBeforeEngine + NoUnwitnessedCleanup + NoPrematurePublication + NoUnconfirmedRetirement + CompletionRequiresCleanup +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/absent-publication-recovery/positive.cfg b/tests/models/tla/absent-publication-recovery/positive.cfg new file mode 100644 index 000000000..db5e420e8 --- /dev/null +++ b/tests/models/tla/absent-publication-recovery/positive.cfg @@ -0,0 +1,12 @@ +CONSTANTS EnforceIntent = TRUE + EnforceBarrier = TRUE + EnforceSelection = TRUE + EnforceCompletion = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK + ForegroundBeforeEngine + NoUnwitnessedCleanup + NoPrematurePublication + NoUnconfirmedRetirement + CompletionRequiresCleanup +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/absent-publication-recovery/stale-selection.cfg b/tests/models/tla/absent-publication-recovery/stale-selection.cfg new file mode 100644 index 000000000..47cec1236 --- /dev/null +++ b/tests/models/tla/absent-publication-recovery/stale-selection.cfg @@ -0,0 +1,12 @@ +CONSTANTS EnforceIntent = TRUE + EnforceBarrier = TRUE + EnforceSelection = FALSE + EnforceCompletion = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK + ForegroundBeforeEngine + NoUnwitnessedCleanup + NoPrematurePublication + NoUnconfirmedRetirement + CompletionRequiresCleanup +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/absent-publication-recovery/unconfirmed-retirement.cfg b/tests/models/tla/absent-publication-recovery/unconfirmed-retirement.cfg new file mode 100644 index 000000000..7e0438c70 --- /dev/null +++ b/tests/models/tla/absent-publication-recovery/unconfirmed-retirement.cfg @@ -0,0 +1,12 @@ +CONSTANTS EnforceIntent = TRUE + EnforceBarrier = TRUE + EnforceSelection = TRUE + EnforceCompletion = FALSE +SPECIFICATION Spec +INVARIANTS TypeOK + ForegroundBeforeEngine + NoUnwitnessedCleanup + NoPrematurePublication + NoUnconfirmedRetirement + CompletionRequiresCleanup +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/absent-publication-recovery/unwitnessed-cleanup.cfg b/tests/models/tla/absent-publication-recovery/unwitnessed-cleanup.cfg new file mode 100644 index 000000000..07f3f71d4 --- /dev/null +++ b/tests/models/tla/absent-publication-recovery/unwitnessed-cleanup.cfg @@ -0,0 +1,12 @@ +CONSTANTS EnforceIntent = FALSE + EnforceBarrier = TRUE + EnforceSelection = TRUE + EnforceCompletion = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK + ForegroundBeforeEngine + NoUnwitnessedCleanup + NoPrematurePublication + NoUnconfirmedRetirement + CompletionRequiresCleanup +CHECK_DEADLOCK FALSE From 4e41bd95b853db6141ef09af7f5585a8f305371a Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 03:18:18 -0400 Subject: [PATCH 6/9] fix(runtime): recover retained graphs after lost reboot publications --- docs/guides/native-candidate.md | 29 + packages/runtime-core/src/graph_cli.rs | 81 ++ packages/runtime-core/src/main.rs | 2 + .../graph/absent_publication_cleanup.rs | 1272 +++++++++++++++++ .../src/provider/graph/bridges/cleanup.rs | 198 ++- .../src/provider/graph/cleanup_enrollment.rs | 1 + .../src/provider/graph/dead_owner_cleanup.rs | 2 +- .../provider/graph/foreground/native_test.rs | 1 + .../absent_publication_recovery.rs | 500 +++++++ .../provider/graph/foreground/transport.rs | 3 + .../src/provider/graph/host_pin_recovery.rs | 8 +- .../src/provider/graph/host_relay.rs | 68 +- .../runtime-core/src/provider/graph/mod.rs | 59 +- .../tests/absent_recovery_cli_contract.rs | 107 ++ 14 files changed, 2320 insertions(+), 11 deletions(-) create mode 100644 packages/runtime-core/src/provider/graph/absent_publication_cleanup.rs create mode 100644 packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs create mode 100644 packages/runtime-core/tests/absent_recovery_cli_contract.rs diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 010e75fff..083e8b35a 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -200,6 +200,35 @@ directory itself during reboot, this command refuses: the old pin receipts no longer exist, and absent pathnames cannot stand in for their recorded owner identities. That case requires a separate selected absence-recovery procedure. +When a **physical host reboot** removed both the deterministic foreground +publication root and this run's relay-control root, inspect the distinct +absence path with the private original provider Owner and its exact +pre-migration host-filesystem inspection: + +```sh +./hack-native --candidate-root /absolute/private/candidate-home graph inspect-absent-publication-cleanup --run-id --original-owner-file --host-inspection-file --json +./hack-native --candidate-root /absolute/private/candidate-home graph recover-absent-publication-cleanup --run-id --original-owner-file --host-inspection-file --expect-selection --retain-data --accept-unpinned-post-reboot --json +``` + +Inspection does not create a publication root. A failed selected action can +leave only its private lock reservation; inspection recognizes that exact +lock-only state. Recovery locks the foreground publication before acquiring +the VM lease, then durably records the selected absence before any cleanup. +It retains volumes, verifies the stopped receipt, and records a separate +absent-publisher retirement. Ordinary publication and cleanup cannot infer +ownership from missing paths. The old foreground PID and original physical +volume are not proved by legacy graph receipts; this path requires explicit +acceptance of that post-reboot limitation and refuses a changed boot, present +or foreign publication, stale selection, pending state, or changed resources. +An interrupted recovery can resume only its exact selected intent on the same +host and immediate guest boot. A later successful restore treats this witness +as history; it never grants cleanup of the new generation. + +This operation does not change historical shared-source device identity. +Source-mounted projects require the separate selected source-continuity step +before normal same-run restore. The command's stopped/data-retained result is +not proof of application startup or routing. + Compatibility is qualified for specific bundle hashes and state formats. The displayed version alone does not establish frontend/executor or downgrade compatibility. V4 and candidate homes remain separate; this procedure does not diff --git a/packages/runtime-core/src/graph_cli.rs b/packages/runtime-core/src/graph_cli.rs index b93a6872a..d962c917f 100644 --- a/packages/runtime-core/src/graph_cli.rs +++ b/packages/runtime-core/src/graph_cli.rs @@ -69,6 +69,87 @@ pub fn command(candidate: &Candidate, args: &[&str]) -> Result (run, original, inspection), + _ => return Err(invalid()), + }; + #[cfg(target_os = "macos")] + { + return graph::inspect_absent_publication_cleanup( + candidate, + run, + Path::new(original), + Path::new(inspection), + ); + } + #[cfg(not(target_os = "macos"))] + { + let _ = (run, original, inspection); + return Err(invalid()); + } + } + if *action == "recover-absent-publication-cleanup" { + let (run, original, inspection, expected) = match *args { + [ + "--run-id", + run, + "--original-owner-file", + original, + "--host-inspection-file", + inspection, + "--expect-selection", + expected, + "--retain-data", + "--accept-unpinned-post-reboot", + ] + | [ + "--run-id", + run, + "--original-owner-file", + original, + "--host-inspection-file", + inspection, + "--expect-selection", + expected, + "--retain-data", + "--accept-unpinned-post-reboot", + "--json", + ] => (run, original, inspection, expected), + _ => return Err(invalid()), + }; + #[cfg(target_os = "macos")] + { + return graph::recover_absent_publication_cleanup( + candidate, + run, + expected, + Path::new(original), + Path::new(inspection), + ); + } + #[cfg(not(target_os = "macos"))] + { + let _ = (run, original, inspection, expected); + return Err(invalid()); + } + } if ["run-selection", "run-service"].contains(action) { return one_off::command(candidate, action, args); } diff --git a/packages/runtime-core/src/main.rs b/packages/runtime-core/src/main.rs index 5878fae07..558aff288 100644 --- a/packages/runtime-core/src/main.rs +++ b/packages/runtime-core/src/main.rs @@ -29,6 +29,8 @@ Usage: hack-local graph retire-recovered-publisher --run-id <32-hex> --expect-owner <32-hex> [--json] hack-local graph inspect-host-pin-recovery --run-id <32-hex> [--json] hack-local graph recover-host-pins --run-id <32-hex> --expect-selection <64-hex> --accept-legacy-device-rebind [--json] + hack-local graph inspect-absent-publication-cleanup --run-id <32-hex> --original-owner-file --host-inspection-file [--json] + hack-local graph recover-absent-publication-cleanup --run-id <32-hex> --original-owner-file --host-inspection-file --expect-selection <64-hex> --retain-data --accept-unpinned-post-reboot [--json] hack-local graph logs --run-id <32-hex> --service [--tail <1..1000>] [--json] hack-local graph exec --run-id <32-hex> --service [--workdir /path] [--timeout-seconds <1..120>] [--json] -- [args...] hack-local graph dependency-plan --dependencies [--json] diff --git a/packages/runtime-core/src/provider/graph/absent_publication_cleanup.rs b/packages/runtime-core/src/provider/graph/absent_publication_cleanup.rs new file mode 100644 index 000000000..ead3beec2 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/absent_publication_cleanup.rs @@ -0,0 +1,1272 @@ +//! Explicit data-retaining recovery when both ephemeral host publications were +//! lost across a physical host reboot. Missing paths never create ordinary +//! ownership authority: the selected, durable witness is the only admission. +use super::{ + Candidate, CandidateError, Engine, Kind, Receipt, dead_owner_cleanup, dependency_slots, + directory, foreground, host_pin_recovery, host_relay, initializer_cache, inspect_resource, + load, startup, state, +}; +use crate::provider::{ + ProjectShareIntent, artifact, host_pin::DeviceRebind, identity, lifecycle, state::Owner, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + fs, + os::unix::fs::MetadataExt, + path::{Path, PathBuf}, +}; + +const INTENT: &str = "absent-publication-cleanup.json"; +const RETIREMENT: &str = "absent-publication-retirement.json"; +const LIMIT: u64 = 2 * 1024 * 1024; + +fn refused() -> CandidateError { + CandidateError::new( + "graph_absent_publication_recovery", + "Selected post-reboot publication absence or retained graph identity changed; evidence and data were preserved.", + ) +} +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} +fn absent(path: &Path) -> Result { + match fs::symlink_metadata(path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true), + Ok(_) => Ok(false), + Err(_) => Err(refused()), + } +} +fn require_absent(path: &Path) -> Result<(), CandidateError> { + if !absent(path)? { + return Err(refused()); + } + Ok(()) +} +fn no_pending(root: &Path) -> Result<(), CandidateError> { + for name in [ + "state.pending", + "dead-owner-cleanup.pending", + "relay-cleanup-bridges.pending", + "one-off.pending", + "one-off-normalization.pending", + "absent-publication-cleanup.pending", + "absent-publication-retirement.pending", + ] { + require_absent(&root.join(name))?; + } + Ok(()) +} +fn retain_interrupted_publications(root: &Path) -> Result<(), CandidateError> { + super::journal::retain_file( + root, + "absent-publication-cleanup.pending", + "absent-publication-cleanup-recovery", + 4 * 1024 * 1024, + )?; + super::journal::retain_file( + root, + "absent-publication-retirement.pending", + "absent-publication-retirement-recovery", + 65536, + )?; + Ok(()) +} +fn private_input(path: &Path, limit: u64) -> Result, CandidateError> { + if !path.is_absolute() || fs::canonicalize(path).map_err(|_| refused())? != path { + return Err(refused()); + } + host_pin_recovery::read_raw(path, limit).map_err(|_| refused()) +} +fn device_only_share( + old: Option<&ProjectShareIntent>, + current: Option<&ProjectShareIntent>, + old_device: u64, + new_device: u64, +) -> bool { + match (old, current) { + (None, None) => true, + (Some(previous), Some(current)) => { + let mut expected = previous.clone(); + expected.device = new_device; + previous.device == old_device && *current == expected + } + _ => false, + } +} + +/// This is the exact serialization shape of the pre-migration inspection. It +/// binds the raw original Owner, one common host-device change and the boot +/// used to approve it; it is not a physical-volume UUID proof. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct FilesystemInspection { + schema: String, + selection_sha256: String, + owner_sha256: String, + machine: String, + host_boot_micros: u64, + provider_start_micros: u64, + old_device: u64, + new_device: u64, + pool_inode: u64, + storage: identity::DiskIdentity, + overlay: identity::DiskIdentity, + project_share: Option, + qualification: String, +} +impl FilesystemInspection { + fn verify(&self, owner_bytes: &[u8], old: &Owner) -> Result<(), CandidateError> { + let mut unsigned = self.clone(); + unsigned.selection_sha256.clear(); + if self.schema != "hack.host-filesystem-recovery/v1" + || self.qualification != "explicit-legacy-migration-original-volume-continuity-unproven" + || self.owner_sha256 != digest(owner_bytes) + || self.selection_sha256 + != digest(&serde_json::to_vec(&unsigned).map_err(|_| refused())?) + || self.machine != old.machine + || self.provider_start_micros != old.process.as_ref().ok_or_else(refused)?.start_micros + || self.old_device == self.new_device + || self.storage.device != self.new_device + || self.overlay.device != self.new_device + || old.storage.as_ref().is_none_or(|disk| { + disk.device != self.old_device + || disk.inode != self.storage.inode + || disk.bytes != self.storage.bytes + || disk.uuid != self.storage.uuid + }) + || old.overlay.as_ref().is_none_or(|disk| { + disk.device != self.old_device + || disk.inode != self.overlay.inode + || disk.bytes != self.overlay.bytes + || disk.uuid != self.overlay.uuid + }) + || !device_only_share( + old.project_share.as_ref(), + self.project_share.as_ref(), + self.old_device, + self.new_device, + ) + { + return Err(refused()); + } + Ok(()) + } +} + +#[cfg(all(test, feature = "environment-launcher"))] +pub(in crate::provider::graph) fn fixture_inspection( + owner_bytes: &[u8], + current: &Owner, + old_device: u64, + host_boot_micros: u64, + pool_inode: u64, +) -> Vec { + let old: Owner = serde_json::from_slice(owner_bytes).unwrap(); + let mut selection = FilesystemInspection { + schema: "hack.host-filesystem-recovery/v1".into(), + selection_sha256: String::new(), + owner_sha256: digest(owner_bytes), + machine: old.machine.clone(), + host_boot_micros, + provider_start_micros: old.process.as_ref().unwrap().start_micros, + old_device, + new_device: current.storage.as_ref().unwrap().device, + pool_inode, + storage: current.storage.clone().unwrap(), + overlay: current.overlay.clone().unwrap(), + project_share: current.project_share.clone(), + qualification: "explicit-legacy-migration-original-volume-continuity-unproven".into(), + }; + selection.selection_sha256 = digest(&serde_json::to_vec(&selection).unwrap()); + serde_json::to_vec_pretty(&selection).unwrap() +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Selection { + version: u8, + candidate: PathBuf, + run: String, + owner: String, + namespace: String, + plan: String, + original_owner_path: PathBuf, + original_owner_sha256: String, + filesystem_inspection_path: PathBuf, + filesystem_inspection_sha256: String, + current_owner_sha256: String, + graph_sha256: String, + host_boot_micros: u64, + old_device: u64, + new_device: u64, + previous_guest_boot: String, + current_guest_boot: String, + foreground_root: PathBuf, + control_root: PathBuf, + source_shared: Option, + environment_inventory: Value, + scoped_bridge_projection: Value, + retained_volumes: BTreeMap, + dependency_reservation: Option, + qualification: String, +} +impl Selection { + pub(super) fn digest(&self) -> Result { + Ok(digest(&serde_json::to_vec(self).map_err(|_| refused())?)) + } + pub(super) fn matches_graph(&self, receipt: &Receipt) -> bool { + self.run == receipt.run + && self.owner == receipt.owner + && self.namespace == receipt.namespace + && self.plan == receipt.plan_id + && self.source_shared == receipt.source.as_ref().and_then(|s| s.shared.clone()) + } + pub(super) fn previous_boot(&self) -> &str { + &self.previous_guest_boot + } + pub(super) fn control_root(&self) -> &Path { + &self.control_root + } +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Intent { + version: u8, + selection_sha256: String, + selection: Selection, + original: Receipt, + environment: Value, + bridges: super::bridges::cleanup::Selection, + #[serde(default, skip_serializing_if = "Option::is_none")] + prior_bridges: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + complete_sha256: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Retirement { + version: u8, + selection_sha256: String, + complete_sha256: String, + owner: String, + run: String, +} + +fn selected_volumes( + engine: &Engine<'_>, + receipt: &Receipt, +) -> Result, CandidateError> { + let mut volumes = BTreeMap::new(); + for (key, resource) in receipt + .resources + .iter() + .filter(|(_, resource)| resource.kind == Kind::Volume) + { + let actual = inspect_resource(engine, receipt, resource)?.ok_or_else(refused)?; + let created = actual["CreatedAt"] + .as_str() + .filter(|value| !value.is_empty()); + if created.is_none() { + return Err(refused()); + } + volumes.insert( + key.clone(), + json!({ + "expected_name": resource.name, + "observed_name": actual["Name"], + "observed_created_at": actual["CreatedAt"], + "observed_mountpoint": actual["Mountpoint"], + "observed_driver": actual["Driver"], + "observed_labels_sha256": digest( + &serde_json::to_vec(&actual["Labels"]).map_err(|_| refused())? + ), + "labels": super::expected_labels(receipt, resource), + }), + ); + } + Ok(volumes) +} + +fn current_owner( + candidate: &Candidate, + old: &Owner, + inspection: &FilesystemInspection, + engine: &Engine<'_>, +) -> Result<(Owner, String), CandidateError> { + lifecycle::host_filesystem::no_auxiliary_update(candidate)?; + if lifecycle::host_filesystem::host_boot_micros()? != inspection.host_boot_micros { + return Err(refused()); + } + let old_process = old.process.as_ref().ok_or_else(refused)?; + // SAFETY: geteuid has no arguments or side effects. + identity::verify( + old_process, + old_process, + &artifact::root(candidate).join("smolvm-bin"), + unsafe { libc::geteuid() }, + ) + .map_err(|_| refused())?; + DeviceRebind { + old: inspection.old_device, + current: inspection.new_device, + } + .definitely_dead_before_boot(old_process, inspection.host_boot_micros) + .map_err(|_| refused())?; + let owner = Owner::load(candidate)?; + lifecycle::verify_disks(candidate, &owner)?; + host_pin_recovery::verify_guest_identity(engine)?; + let pool = + fs::symlink_metadata(candidate.state_root.join("run/smolvm")).map_err(|_| refused())?; + if old.phase != "running" + || owner.phase != "running" + || old.token != owner.token + || owner.token != engine.guest().incarnation() + || old.guest_boot_id.as_deref() != owner.previous_guest_boot_id.as_deref() + || owner.guest_boot_id.as_deref() != Some(engine.guest().boot_id()) + || old.guest_boot_id.as_deref() == owner.guest_boot_id.as_deref() + || owner.storage.as_ref() != Some(&inspection.storage) + || owner.overlay.as_ref() != Some(&inspection.overlay) + || owner.project_share != inspection.project_share + || pool.ino() != inspection.pool_inode + || pool.dev() != inspection.new_device + { + return Err(refused()); + } + let mut expected = old.clone(); + expected.storage = owner.storage.clone(); + expected.overlay = owner.overlay.clone(); + expected.project_share = owner.project_share.clone(); + expected.process = owner.process.clone(); + expected.guest_boot_id = owner.guest_boot_id.clone(); + expected.previous_guest_boot_id = owner.previous_guest_boot_id.clone(); + expected.daemon_pid = owner.daemon_pid; + expected.daemon_start = owner.daemon_start; + if expected != owner { + return Err(refused()); + } + let owner_bytes = host_pin_recovery::read_raw( + &candidate.state_root.join("run/smolvm/owner.json"), + 1024 * 1024, + )?; + let raw_owner: Owner = serde_json::from_slice(&owner_bytes).map_err(|_| refused())?; + if raw_owner != owner { + return Err(refused()); + } + let owner_sha256 = digest(&owner_bytes); + Ok((owner, owner_sha256)) +} + +fn publication_roots(selection: &Selection, allow_lock_only: bool) -> Result<(), CandidateError> { + require_absent(&selection.control_root)?; + match fs::symlink_metadata(&selection.foreground_root) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound && !allow_lock_only => Ok(()), + Ok(_) if allow_lock_only => { + state::check_private_directory(&selection.foreground_root)?; + let mut entries = fs::read_dir(&selection.foreground_root).map_err(|_| refused())?; + let entry = entries.next().ok_or_else(refused)?.map_err(|_| refused())?; + if entry.file_name() != "operation.lock" || entries.next().is_some() { + return Err(refused()); + } + let lock = fs::symlink_metadata(entry.path()).map_err(|_| refused())?; + // SAFETY: geteuid has no arguments or side effects. + if !lock.is_file() + || lock.nlink() != 1 + || lock.mode() & 0o7777 != 0o600 + || lock.uid() != unsafe { libc::geteuid() } + { + return Err(refused()); + } + Ok(()) + } + _ => Err(refused()), + } +} + +fn select_content( + candidate: &Candidate, + run: &str, + original_owner_path: &Path, + filesystem_inspection_path: &Path, + engine: &Engine<'_>, + allow_lock_only: bool, +) -> Result<(Selection, Receipt), CandidateError> { + let (receipt, root) = load(candidate, engine, run)?; + no_pending(&root)?; + if receipt.phase != "ready-observed" + || receipt.relay_cleanup.is_some() + || receipt.relay_startup.is_none() + || !super::hex(&receipt.owner, 32) + { + return Err(refused()); + } + initializer_cache::require_resolved(&receipt)?; + startup::require_dependency_rebind_complete(&root, &receipt)?; + dead_owner_cleanup::require_historical_recovery(&root, &receipt)?; + let old_bytes = private_input(original_owner_path, 1024 * 1024)?; + let old: Owner = serde_json::from_slice(&old_bytes).map_err(|_| refused())?; + let inspection_bytes = private_input(filesystem_inspection_path, 64 * 1024)?; + let inspection: FilesystemInspection = + serde_json::from_slice(&inspection_bytes).map_err(|_| refused())?; + inspection.verify(&old_bytes, &old)?; + let (owner, current_owner_sha256) = current_owner(candidate, &old, &inspection, engine)?; + if old.checkout != candidate.checkout + || old.machine != inspection.machine + || owner.machine != inspection.machine + || old + .project_share + .as_ref() + .is_some_and(|share| share.device != inspection.old_device) + || receipt + .source + .as_ref() + .and_then(|source| source.shared.as_ref()) + != old.project_share.as_ref() + { + return Err(refused()); + } + let startup = receipt.relay_startup.as_ref().ok_or_else(refused)?; + let foreground_root = foreground::transport::root(candidate, run)?; + let graph_bytes = host_pin_recovery::read_raw(&root.join("state.json"), LIMIT)?; + if graph_bytes != serde_json::to_vec_pretty(&receipt).map_err(|_| refused())? { + return Err(refused()); + } + let graph_sha256 = digest(&graph_bytes); + let rebind = DeviceRebind { + old: inspection.old_device, + current: inspection.new_device, + }; + let dependency_reservation = + dependency_slots::inspect_legacy(candidate, run, rebind, inspection.host_boot_micros)?; + let previous_guest_boot = old.guest_boot_id.clone().ok_or_else(refused)?; + let selection = Selection { + version: 1, + candidate: candidate.checkout.clone(), + run: run.into(), + owner: receipt.owner.clone(), + namespace: receipt.namespace.clone(), + plan: receipt.plan_id.clone(), + original_owner_path: original_owner_path.to_path_buf(), + original_owner_sha256: digest(&old_bytes), + filesystem_inspection_path: filesystem_inspection_path.to_path_buf(), + filesystem_inspection_sha256: digest(&inspection_bytes), + current_owner_sha256, + graph_sha256, + host_boot_micros: inspection.host_boot_micros, + old_device: inspection.old_device, + new_device: inspection.new_device, + previous_guest_boot: previous_guest_boot.clone(), + current_guest_boot: owner.guest_boot_id.ok_or_else(refused)?, + foreground_root, + control_root: startup.control_root.clone(), + source_shared: receipt + .source + .as_ref() + .and_then(|source| source.shared.clone()), + environment_inventory: serde_json::to_value(super::environment::cleanup_inventory( + candidate, engine, &receipt, &root, + )?) + .map_err(|_| refused())?, + scoped_bridge_projection: super::bridges::cleanup::scoped_absence_projection( + candidate, + engine, + &receipt, + &previous_guest_boot, + )?, + retained_volumes: selected_volumes(engine, &receipt)?, + dependency_reservation, + qualification: "explicit-unpinned-post-host-reboot-original-volume-continuity-unproven" + .into(), + }; + publication_roots(&selection, allow_lock_only)?; + host_pin_recovery::verify_volume_projections(engine, &receipt, &selection.retained_volumes)?; + for resource in receipt.resources.values() { + if resource.kind != Kind::Volume && inspect_resource(engine, &receipt, resource)?.is_none() + { + return Err(refused()); + } + } + Ok((selection, receipt)) +} + +pub fn inspect( + candidate: &Candidate, + run: &str, + original_owner_path: &Path, + filesystem_inspection_path: &Path, +) -> Result { + let existing = Reservation::inspect_existing(candidate, run)?; + let engine = Engine::connect_cleanup_wait(candidate)?; + let (selected, _) = select_content( + candidate, + run, + original_owner_path, + filesystem_inspection_path, + &engine, + existing.is_some(), + )?; + if let Some(reservation) = &existing { + reservation.verify()?; + } + Ok(json!({ + "run": run, + "selection_sha256": selected.digest()?, + "host_boot_micros": selected.host_boot_micros, + "qualification": selected.qualification, + "data_retained": true, + })) +} + +struct Reservation { + root: PathBuf, + identity: (u64, u64), + lock: state::Lock, +} +impl Reservation { + fn inspect_existing(candidate: &Candidate, run: &str) -> Result, CandidateError> { + let root = foreground::transport::root(candidate, run)?; + match fs::symlink_metadata(&root) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Ok(_) => { + state::check_private_directory(&root)?; + let lock = state::Lock::acquire_existing(&root)?; + let metadata = fs::symlink_metadata(&root).map_err(|_| refused())?; + let value = Self { + root, + identity: (metadata.dev(), metadata.ino()), + lock, + }; + value.verify()?; + Ok(Some(value)) + } + Err(_) => Err(refused()), + } + } + fn acquire(candidate: &Candidate, run: &str) -> Result { + let root = foreground::transport::root(candidate, run)?; + let lock = match fs::symlink_metadata(&root) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + state::Lock::acquire(&root)? + } + Ok(_) => { + state::check_private_directory(&root)?; + state::Lock::acquire_existing(&root)? + } + Err(_) => return Err(refused()), + }; + let metadata = fs::symlink_metadata(&root).map_err(|_| refused())?; + let value = Self { + root, + identity: (metadata.dev(), metadata.ino()), + lock, + }; + value.verify()?; + Ok(value) + } + fn verify(&self) -> Result<(), CandidateError> { + let metadata = fs::symlink_metadata(&self.root).map_err(|_| refused())?; + if !metadata.is_dir() + || (metadata.dev(), metadata.ino()) != self.identity + || state::check_private_directory(&self.root).is_err() + { + return Err(refused()); + } + host_pin_recovery::exact_lock_path(&self.root, &self.lock).map_err(|_| refused())?; + let lock = fs::symlink_metadata(self.root.join("operation.lock")).map_err(|_| refused())?; + // SAFETY: geteuid has no arguments or side effects. + if lock.mode() & 0o7777 != 0o600 + || lock.uid() != unsafe { libc::geteuid() } + || lock.nlink() != 1 + { + return Err(refused()); + } + // This verifier is intentionally limited to the reserved root; it + // does not infer control-root or graph ownership from the lock. + let mut entries = fs::read_dir(&self.root).map_err(|_| refused())?; + let entry = entries.next().ok_or_else(refused)?.map_err(|_| refused())?; + if entry.file_name() != "operation.lock" || entries.next().is_some() { + return Err(refused()); + } + Ok(()) + } +} + +fn verify_static_with( + candidate: &Candidate, + run: &str, + engine: &Engine<'_>, + selected: &Selection, + original: &Receipt, + foreground_root: &Path, + verify_lock: impl Fn() -> Result<(), CandidateError>, +) -> Result { + if selected.version != 1 + || selected.candidate != candidate.checkout + || selected.run != run + || selected.foreground_root != foreground_root + || selected.qualification + != "explicit-unpinned-post-host-reboot-original-volume-continuity-unproven" + || !selected.matches_graph(original) + || selected.digest()?.len() != 64 + { + return Err(refused()); + } + verify_lock()?; + publication_roots(selected, true)?; + lifecycle::host_filesystem::no_auxiliary_update(candidate)?; + if lifecycle::host_filesystem::host_boot_micros()? != selected.host_boot_micros { + return Err(refused()); + } + let old_bytes = private_input(&selected.original_owner_path, 1024 * 1024)?; + let inspection_bytes = private_input(&selected.filesystem_inspection_path, 64 * 1024)?; + if digest(&old_bytes) != selected.original_owner_sha256 + || digest(&inspection_bytes) != selected.filesystem_inspection_sha256 + { + return Err(refused()); + } + let old: Owner = serde_json::from_slice(&old_bytes).map_err(|_| refused())?; + let inspection: FilesystemInspection = + serde_json::from_slice(&inspection_bytes).map_err(|_| refused())?; + inspection.verify(&old_bytes, &old)?; + let (owner, owner_sha256) = current_owner(candidate, &old, &inspection, engine)?; + if owner_sha256 != selected.current_owner_sha256 + || owner.token != selected.owner + || owner.guest_boot_id.as_deref() != Some(&selected.current_guest_boot) + || owner.previous_guest_boot_id.as_deref() != Some(&selected.previous_guest_boot) + || inspection.old_device != selected.old_device + || inspection.new_device != selected.new_device + { + return Err(refused()); + } + let (receipt, root) = load(candidate, engine, run)?; + no_pending(&root)?; + let receipt_bytes = host_pin_recovery::read_raw(&root.join("state.json"), LIMIT)?; + if receipt_bytes != serde_json::to_vec_pretty(&receipt).map_err(|_| refused())? { + return Err(refused()); + } + if dead_owner_cleanup::immutable(&receipt)? != dead_owner_cleanup::immutable(original)? + || !selected.matches_graph(&receipt) + || !["ready-observed", "cleanup-intent", "stopped-data-retained"] + .contains(&receipt.phase.as_str()) + { + return Err(refused()); + } + if receipt.phase == "ready-observed" && digest(&receipt_bytes) != selected.graph_sha256 { + return Err(refused()); + } + if receipt.phase == "ready-observed" + && (serde_json::to_value(super::environment::cleanup_inventory( + candidate, engine, &receipt, &root, + )?) + .map_err(|_| refused())? + != selected.environment_inventory + || super::bridges::cleanup::scoped_absence_projection( + candidate, + engine, + &receipt, + &selected.previous_guest_boot, + )? != selected.scoped_bridge_projection) + { + return Err(refused()); + } + host_pin_recovery::verify_volume_projections(engine, &receipt, &selected.retained_volumes)?; + let rebind = DeviceRebind { + old: selected.old_device, + current: selected.new_device, + }; + if let Some(legacy) = &selected.dependency_reservation { + dependency_slots::verify_legacy_remaining( + candidate, + run, + rebind, + legacy, + receipt.phase != "ready-observed", + )?; + } else if dependency_slots::inspect_legacy(candidate, run, rebind, selected.host_boot_micros)? + .is_some() + { + return Err(refused()); + } + verify_lock()?; + Ok(receipt) +} + +fn verify_static( + candidate: &Candidate, + run: &str, + engine: &Engine<'_>, + selected: &Selection, + original: &Receipt, + reservation: &Reservation, +) -> Result { + verify_static_with( + candidate, + run, + engine, + selected, + original, + &reservation.root, + || reservation.verify(), + ) +} + +fn read_intent(root: &Path) -> Result, CandidateError> { + let path = root.join(INTENT); + if absent(&path)? { + return Ok(None); + } + let intent: Intent = state::read_bounded(&path, 4 * 1024 * 1024).map_err(|_| refused())?; + if intent.version != 1 + || intent.selection_sha256 != intent.selection.digest()? + || intent.selection.graph_sha256 + != digest(&serde_json::to_vec_pretty(&intent.original).map_err(|_| refused())?) + || !intent.selection.matches_graph(&intent.original) + { + return Err(refused()); + } + Ok(Some(intent)) +} +fn completed_receipt( + candidate: &Candidate, + engine: &Engine<'_>, + intent: &Intent, + current: &Receipt, + root: &Path, +) -> Result { + if current.phase != "stopped-data-retained" { + return Err(refused()); + } + let environment = super::environment::cleanup_inventory(candidate, engine, current, root)?; + if serde_json::to_value(&environment).map_err(|_| refused())? != intent.environment { + return Err(refused()); + } + super::bridges::cleanup::verify_recovery_file( + root, + &intent.bridges, + intent.prior_bridges.as_ref(), + )?; + host_relay::inspect_cleanup_absence( + candidate, + engine, + current, + &environment, + &intent.bridges, + &intent.selection, + )?; + Ok(digest(&host_pin_recovery::read_raw( + &root.join("state.json"), + LIMIT, + )?)) +} + +fn retire( + candidate: &Candidate, + engine: &Engine<'_>, + intent: &Intent, + current: &Receipt, + root: &Path, + reservation: &Reservation, +) -> Result<(), CandidateError> { + reservation.verify()?; + let complete = intent.complete_sha256.as_deref().ok_or_else(refused)?; + if completed_receipt(candidate, engine, intent, current, root)? != complete { + return Err(refused()); + } + let retirement = Retirement { + version: 1, + selection_sha256: intent.selection_sha256.clone(), + complete_sha256: complete.into(), + owner: intent.selection.owner.clone(), + run: intent.selection.run.clone(), + }; + let path = root.join(RETIREMENT); + if absent(&path)? { + verify_static( + candidate, + &intent.selection.run, + engine, + &intent.selection, + &intent.original, + reservation, + )?; + state::write(&path, &retirement)?; + } else { + let recorded: Retirement = state::read_bounded(&path, 65536).map_err(|_| refused())?; + if recorded.version != retirement.version + || recorded.selection_sha256 != retirement.selection_sha256 + || recorded.complete_sha256 != retirement.complete_sha256 + || recorded.owner != retirement.owner + || recorded.run != retirement.run + { + return Err(refused()); + } + } + reservation.verify() +} + +/// A pending absence intent blocks ordinary publication. Completed retirement +/// permits only the explicit restored-generation bind; it is not a Pin. +pub(super) fn publication_allowed( + candidate: &Candidate, + run: &str, + retired: bool, +) -> Result<(), CandidateError> { + let root = directory(candidate, run)?; + require_absent(&root.join("absent-publication-cleanup.pending"))?; + require_absent(&root.join("absent-publication-retirement.pending"))?; + if absent(&root.join(INTENT))? { + return Ok(()); + } + if !retired { + return Err(refused()); + } + let intent = read_intent(&root)?.ok_or_else(refused)?; + let complete = intent.complete_sha256.as_deref().ok_or_else(refused)?; + let retirement: Retirement = + state::read_bounded(&root.join(RETIREMENT), 65536).map_err(|_| refused())?; + let current: Receipt = state::read_bounded(&root.join("state.json"), LIMIT)?; + if retirement.version != 1 + || retirement.selection_sha256 != intent.selection_sha256 + || retirement.complete_sha256 != complete + || retirement.owner != current.owner + || retirement.run != run + || current.phase != "stopped-data-retained" + { + return Err(refused()); + } + let current_sha256 = digest(&host_pin_recovery::read_raw( + &root.join("state.json"), + LIMIT, + )?); + if current_sha256 != complete && retained(&root, ¤t)? { + return Err(refused()); + } + Ok(()) +} + +/// Retention is bound to the completed stopped receipt, not to missing paths. +pub(super) fn retained(root: &Path, receipt: &Receipt) -> Result { + let Some(intent) = read_intent(root)? else { + return Ok(false); + }; + let Some(complete) = intent.complete_sha256.as_deref() else { + return Err(refused()); + }; + let retirement: Retirement = + state::read_bounded(&root.join(RETIREMENT), 65536).map_err(|_| refused())?; + let current: Receipt = state::read_bounded(&root.join("state.json"), LIMIT)?; + let current_sha256 = digest(&host_pin_recovery::read_raw( + &root.join("state.json"), + LIMIT, + )?); + let requested_sha256 = digest(&serde_json::to_vec_pretty(receipt).map_err(|_| refused())?); + if current_sha256 != requested_sha256 + || receipt.phase != "stopped-data-retained" + || retirement.version != 1 + || retirement.selection_sha256 != intent.selection_sha256 + || retirement.complete_sha256 != complete + || retirement.run != receipt.run + || retirement.owner != receipt.owner + || serde_json::to_vec_pretty(¤t).map_err(|_| refused())? + != serde_json::to_vec_pretty(receipt).map_err(|_| refused())? + { + return Err(refused()); + } + if requested_sha256 != complete { + if intent.original.run != receipt.run + || intent.original.owner != receipt.owner + || intent.original.namespace != receipt.namespace + || !super::restore_history::confirms_prior_generation(root, receipt)? + { + return Err(refused()); + } + super::cleanup_enrollment::retention_receipt(receipt, false)?; + return Ok(false); + } + Ok(true) +} + +/// Exact completed, retired first-generation proof for a later independently +/// selected source-continuity transition. The proof grants no authority for a +/// subsequent graph generation and does not change the original graph source. +#[derive(Clone, Serialize)] +#[allow(dead_code)] // Consumed by the independently selected source-continuity unit. +pub(super) struct CompletedSourceProof { + pub original_ready_sha256: String, + pub completed_stopped_sha256: String, + pub intent_raw_sha256: String, + pub retirement_raw_sha256: String, + pub original_owner_sha256: String, + pub current_owner_sha256: String, + pub host_boot_micros: u64, + pub previous_guest_boot: String, + pub current_guest_boot: String, + pub old_share: Option, + pub current_share: Option, + pub retained_volumes: BTreeMap, +} + +/// The caller holds the foreground retirement lock before its Engine lease and +/// must recheck this proof at the final publication boundary. A historical +/// sidecar with a different stopped receipt cannot lend authority. +#[allow(dead_code)] // Exposed for the subsequent source-continuity unit. +pub(super) fn verify_completed_under( + candidate: &Candidate, + engine: &Engine<'_>, + run: &str, + receipt: &Receipt, + retired: &foreground::transport::Retired, +) -> Result, CandidateError> { + let root = directory(candidate, run)?; + if absent(&root.join(INTENT))? { + return Ok(None); + } + retired.verify()?; + let intent = read_intent(&root)?.ok_or_else(refused)?; + let complete = intent.complete_sha256.as_deref().ok_or_else(refused)?; + let (current, _) = load(candidate, engine, run)?; + if current.phase != "stopped-data-retained" + || digest(&host_pin_recovery::read_raw( + &root.join("state.json"), + LIMIT, + )?) != complete + || serde_json::to_vec_pretty(¤t).map_err(|_| refused())? + != serde_json::to_vec_pretty(receipt).map_err(|_| refused())? + { + return Err(refused()); + } + let foreground_root = foreground::transport::root(candidate, run)?; + verify_static_with( + candidate, + run, + engine, + &intent.selection, + &intent.original, + &foreground_root, + || retired.verify(), + )?; + if completed_receipt(candidate, engine, &intent, ¤t, &root)? != complete { + return Err(refused()); + } + let retirement_bytes = host_pin_recovery::read_raw(&root.join(RETIREMENT), 65536)?; + let retirement: Retirement = + serde_json::from_slice(&retirement_bytes).map_err(|_| refused())?; + if retirement.version != 1 + || retirement.selection_sha256 != intent.selection_sha256 + || retirement.complete_sha256 != complete + || retirement.run != run + || retirement.owner != current.owner + { + return Err(refused()); + } + let old_bytes = private_input(&intent.selection.original_owner_path, 1024 * 1024)?; + let old: Owner = serde_json::from_slice(&old_bytes).map_err(|_| refused())?; + let inspection_bytes = private_input(&intent.selection.filesystem_inspection_path, 64 * 1024)?; + let inspection: FilesystemInspection = + serde_json::from_slice(&inspection_bytes).map_err(|_| refused())?; + inspection.verify(&old_bytes, &old)?; + retired.verify()?; + Ok(Some(CompletedSourceProof { + original_ready_sha256: intent.selection.graph_sha256.clone(), + completed_stopped_sha256: complete.into(), + intent_raw_sha256: digest(&host_pin_recovery::read_raw( + &root.join(INTENT), + 4 * 1024 * 1024, + )?), + retirement_raw_sha256: digest(&retirement_bytes), + original_owner_sha256: intent.selection.original_owner_sha256.clone(), + current_owner_sha256: intent.selection.current_owner_sha256.clone(), + host_boot_micros: intent.selection.host_boot_micros, + previous_guest_boot: intent.selection.previous_guest_boot.clone(), + current_guest_boot: intent.selection.current_guest_boot.clone(), + old_share: old.project_share, + current_share: inspection.project_share, + retained_volumes: intent.selection.retained_volumes.clone(), + })) +} + +/// Explicit, hash-selected cleanup and separate durable absent-publisher +/// retirement. This command never removes a selected graph volume. +pub fn recover( + candidate: &Candidate, + run: &str, + expected: &str, + original_owner_path: &Path, + filesystem_inspection_path: &Path, +) -> Result { + if !super::hex(expected, 64) { + return Err(refused()); + } + // Foreground publication takes this lock before the VM operation lease. + // Following the same order prevents a new publisher from racing admission. + let reservation = Reservation::acquire(candidate, run)?; + let engine = Engine::connect_cleanup_wait(candidate)?; + let root = directory(candidate, run)?; + retain_interrupted_publications(&root)?; + let mut intent = if let Some(existing) = read_intent(&root)? { + if existing.selection_sha256 != expected + || existing.selection.original_owner_path != original_owner_path + || existing.selection.filesystem_inspection_path != filesystem_inspection_path + { + return Err(refused()); + } + existing + } else { + let (selection, original) = select_content( + candidate, + run, + original_owner_path, + filesystem_inspection_path, + &engine, + true, + )?; + if selection.digest()? != expected { + return Err(refused()); + } + let environment = + super::environment::cleanup_inventory(candidate, &engine, &original, &root)?; + let environment_value = serde_json::to_value(&environment).map_err(|_| refused())?; + if environment_value != selection.environment_inventory { + return Err(refused()); + } + host_relay::cleanup_preflight(&engine, &original, &root, false)?; + let bridges = super::bridges::cleanup::capture_previous_boot_absence( + candidate, &engine, &original, &selection, + )?; + if super::bridges::cleanup::selection_absence_projection(&bridges)? + != selection.scoped_bridge_projection + { + return Err(refused()); + } + let prior_bridges = + super::bridges::cleanup::capture_prior_generation(&root, &bridges, &original)?; + if prior_bridges.is_some() + && !super::restore_history::confirms_prior_generation(&root, &original)? + { + return Err(refused()); + } + let selected = Intent { + version: 1, + selection_sha256: expected.into(), + selection, + original, + environment: environment_value, + bridges, + prior_bridges, + complete_sha256: None, + }; + reservation.verify()?; + state::write(&root.join(INTENT), &selected)?; + #[cfg(test)] + super::fault_pause(&root, run, "absent-after-intent")?; + selected + }; + let current = verify_static( + candidate, + run, + &engine, + &intent.selection, + &intent.original, + &reservation, + )?; + if intent.complete_sha256.is_none() { + if current.phase == "stopped-data-retained" { + // Cleanup may have completed immediately before the intent's final + // commit. Exact confirmation permits a deterministic retry. + intent.complete_sha256 = Some(completed_receipt( + candidate, &engine, &intent, ¤t, &root, + )?); + state::write(&root.join(INTENT), &intent)?; + } else { + super::bridges::cleanup::verify_remaining_absence( + candidate, + &engine, + ¤t, + &intent.bridges, + &intent.selection, + )?; + super::bridges::cleanup::verify_recovery_file( + &root, + &intent.bridges, + intent.prior_bridges.as_ref(), + )?; + super::bridges::cleanup::recover_persist(&root, &intent.bridges)?; + reservation.verify()?; + let cleaned = super::cleanup_owned_fenced( + candidate, + &engine, + current, + &root, + false, + true, + || { + verify_static( + candidate, + run, + &engine, + &intent.selection, + &intent.original, + &reservation, + ) + .map(|_| ()) + }, + )?; + reservation.verify()?; + intent.complete_sha256 = Some(completed_receipt( + candidate, &engine, &intent, &cleaned, &root, + )?); + state::write(&root.join(INTENT), &intent)?; + } + } + let (current, _) = load(candidate, &engine, run)?; + let complete = completed_receipt(candidate, &engine, &intent, ¤t, &root)?; + if intent.complete_sha256.as_deref() != Some(complete.as_str()) { + return Err(refused()); + } + if let Some(reservation_record) = &intent.selection.dependency_reservation { + dependency_slots::recover_cleaned( + candidate, + ¤t, + Some(( + DeviceRebind { + old: intent.selection.old_device, + current: intent.selection.new_device, + }, + reservation_record, + )), + )?; + } else { + dependency_slots::recover_cleaned(candidate, ¤t, None)?; + } + retire(candidate, &engine, &intent, ¤t, &root, &reservation)?; + Ok(json!({ + "run": run, + "phase": "stopped-data-retained", + "publisher_retired": true, + "data_retained": true, + "selection_sha256": expected, + "qualification": intent.selection.qualification, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + fn candidate() -> (super::super::tests::Fixture, Candidate) { + let fixture = super::super::tests::Fixture::new(); + let candidate = Candidate::discover(&fixture.0).unwrap(); + (fixture, candidate) + } + + #[test] + fn read_only_inspection_of_existing_reservation_never_creates_a_root() { + let (_fixture, candidate) = candidate(); + let run = "a".repeat(32); + let root = foreground::transport::root(&candidate, &run).unwrap(); + assert!( + Reservation::inspect_existing(&candidate, &run) + .unwrap() + .is_none() + ); + assert!(absent(&root).unwrap()); + let acquired = Reservation::acquire(&candidate, &run).unwrap(); + acquired.verify().unwrap(); + drop(acquired); + let inspected = Reservation::inspect_existing(&candidate, &run) + .unwrap() + .unwrap(); + inspected.verify().unwrap(); + assert_eq!( + fs::read_dir(&root).unwrap().count(), + 1, + "only the lock-only reservation is present" + ); + drop(inspected); + fs::remove_file(root.join("operation.lock")).unwrap(); + fs::remove_dir(root).unwrap(); + } + + #[test] + fn substituted_foreground_lock_path_refuses_without_deleting_replacement() { + let (_fixture, candidate) = candidate(); + let run = "b".repeat(32); + let acquired = Reservation::acquire(&candidate, &run).unwrap(); + let root = acquired.root.clone(); + let path = acquired.root.join("operation.lock"); + let saved = acquired.root.join("operation.held"); + fs::rename(&path, &saved).unwrap(); + fs::write(&path, b"foreign replacement").unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + let replacement = fs::symlink_metadata(&path).unwrap(); + assert!(acquired.verify().is_err()); + let after = fs::symlink_metadata(&path).unwrap(); + assert_eq!( + (after.dev(), after.ino()), + (replacement.dev(), replacement.ino()) + ); + assert_eq!(fs::read(&path).unwrap(), b"foreign replacement"); + drop(acquired); + fs::remove_file(&path).unwrap(); + fs::rename(saved, &path).unwrap(); + fs::remove_file(&path).unwrap(); + fs::remove_dir(&root).unwrap(); + } + + #[test] + fn interrupted_intent_blocks_every_ordinary_publisher_mode() { + let (_fixture, candidate) = candidate(); + let run = "c".repeat(32); + let root = directory(&candidate, &run).unwrap(); + state::private_directory(&root).unwrap(); + fs::write(root.join("absent-publication-cleanup.pending"), b"partial").unwrap(); + assert!(publication_allowed(&candidate, &run, false).is_err()); + assert!(publication_allowed(&candidate, &run, true).is_err()); + assert!(absent(&root.join(INTENT)).unwrap()); + } + + #[test] + fn legacy_share_rebind_refuses_every_non_device_change() { + let old = ProjectShareIntent { + project: PathBuf::from("/private/tmp/selected"), + guest_path: "/workspace".into(), + device: 10, + inode: 92, + unfiltered_source: true, + }; + let mut current = old.clone(); + current.device = 11; + assert!(device_only_share(Some(&old), Some(¤t), 10, 11)); + assert!(!device_only_share(None, Some(¤t), 10, 11)); + assert!(!device_only_share(Some(&old), None, 10, 11)); + assert!(!device_only_share(Some(&old), Some(¤t), 9, 11)); + for changed in [ + { + let mut value = current.clone(); + value.project = PathBuf::from("/private/tmp/other"); + value + }, + { + let mut value = current.clone(); + value.guest_path = "/different".into(); + value + }, + { + let mut value = current.clone(); + value.inode += 1; + value + }, + { + let mut value = current.clone(); + value.unfiltered_source = false; + value + }, + ] { + assert!(!device_only_share(Some(&old), Some(&changed), 10, 11)); + } + } +} diff --git a/packages/runtime-core/src/provider/graph/bridges/cleanup.rs b/packages/runtime-core/src/provider/graph/bridges/cleanup.rs index c76005ce1..17779815f 100644 --- a/packages/runtime-core/src/provider/graph/bridges/cleanup.rs +++ b/packages/runtime-core/src/provider/graph/bridges/cleanup.rs @@ -11,12 +11,21 @@ pub(crate) struct Selection { previous_boot: Option, #[serde(default, skip_serializing_if = "Option::is_none")] predecessor_owner: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + predecessor_absence: Option, run: String, plan: String, capacity: u8, serial: u64, selected: BTreeMap, } +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct AbsentPredecessor { + selection_sha256: String, + previous_boot: String, + control_root: PathBuf, +} #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -54,6 +63,15 @@ fn validate_selection_recovery( receipt: &Receipt, selection: &Selection, host_pin: Option<&super::super::host_pin_recovery::Witness>, +) -> Result<(), CandidateError> { + validate_selection_with_absence(engine, receipt, selection, host_pin, None) +} +fn validate_selection_with_absence( + engine: &Engine<'_>, + receipt: &Receipt, + selection: &Selection, + host_pin: Option<&super::super::host_pin_recovery::Witness>, + absence: Option<&super::super::absent_publication_cleanup::Selection>, ) -> Result<(), CandidateError> { let capacity = engine.guest().bridge_intent().map_or(0, |v| v.slots); if selection.version != 1 @@ -81,6 +99,24 @@ fn validate_selection_recovery( return Err(invalid()); } } + match (&selection.predecessor_absence, absence) { + (None, None) => {} + (None, Some(witness)) + if !selection.selected.is_empty() + && selection.predecessor_owner.is_none() + && selection.previous_boot.as_deref() == Some(witness.previous_boot()) + && witness.matches_graph(receipt) => {} + (Some(recorded), Some(witness)) + if recorded.selection_sha256 == witness.digest()? + && recorded.previous_boot == witness.previous_boot() + && selection.previous_boot.as_deref() == Some(witness.previous_boot()) + && recorded.control_root == witness.control_root() + && witness.matches_graph(receipt) + && selection.selected.is_empty() + && selection.predecessor_owner.is_none() + && pinned_predecessor_eligible(receipt) => {} + _ => return Err(invalid()), + } Ok(()) } @@ -92,7 +128,9 @@ fn validate_bindings( if let Some(previous) = &selection.previous_boot { if previous == &selection.boot || previous_boot != Some(previous.as_str()) - || (selection.selected.is_empty() && selection.predecessor_owner.is_none()) + || (selection.selected.is_empty() + && selection.predecessor_owner.is_none() + && selection.predecessor_absence.is_none()) { return Err(invalid()); } @@ -105,6 +143,23 @@ fn validate_bindings( }) { return Err(invalid()); } + if selection + .predecessor_absence + .as_ref() + .is_some_and(|absence| { + !hex(&absence.selection_sha256, 64) + || absence.previous_boot != selection.previous_boot.as_deref().unwrap_or_default() + || receipt + .relay_startup + .as_ref() + .is_none_or(|startup| startup.control_root != absence.control_root) + || selection.predecessor_owner.is_some() + || !selection.selected.is_empty() + || !pinned_predecessor_eligible(receipt) + }) + { + return Err(invalid()); + } let store = Store { version: 1, owner: selection.owner.clone(), @@ -153,6 +208,7 @@ pub(crate) fn capture( boot: engine.guest().boot_id().into(), previous_boot: None, predecessor_owner: None, + predecessor_absence: None, run: receipt.run.clone(), plan: receipt.plan_id.clone(), capacity: engine.guest().bridge_intent().map_or(0, |v| v.slots), @@ -248,6 +304,7 @@ pub(crate) fn capture_previous_boot( boot: engine.guest().boot_id().into(), previous_boot: Some(previous.into()), predecessor_owner: None, + predecessor_absence: None, run: receipt.run.clone(), plan: receipt.plan_id.clone(), capacity: engine.guest().bridge_intent().map_or(0, |v| v.slots), @@ -280,6 +337,7 @@ pub(crate) fn capture_previous_boot( || prior.boot != previous || prior.previous_boot.is_some() || prior.predecessor_owner.is_some() + || prior.predecessor_absence.is_some() || prior.run != selection.run || prior.plan != selection.plan || prior.capacity != selection.capacity @@ -310,6 +368,106 @@ pub(crate) fn capture_previous_boot( Ok(selection) } +/// A missing relay-control root has no Pin fingerprint. Select a distinct, +/// witness-bound predecessor only with the exact current bridge registry. +pub(crate) fn capture_previous_boot_absence( + candidate: &Candidate, + engine: &Engine<'_>, + receipt: &Receipt, + witness: &super::super::absent_publication_cleanup::Selection, +) -> Result { + let store = strict_store(candidate, engine)?; + let mut selection = Selection { + version: 1, + owner: engine.guest().incarnation().into(), + boot: engine.guest().boot_id().into(), + previous_boot: Some(witness.previous_boot().into()), + predecessor_owner: None, + predecessor_absence: None, + run: receipt.run.clone(), + plan: receipt.plan_id.clone(), + capacity: engine + .guest() + .bridge_intent() + .map_or(0, |value| value.slots), + serial: store.next_launch_serial, + selected: store + .slots + .into_iter() + .filter(|(_, assignment)| assignment.run == receipt.run) + .map(|(slot, assignment)| { + ( + slot, + Selected { + assignment, + helper: None, + }, + ) + }) + .collect(), + }; + if selection.selected.is_empty() { + selection.predecessor_absence = Some(AbsentPredecessor { + selection_sha256: witness.digest()?, + previous_boot: witness.previous_boot().into(), + control_root: witness.control_root().to_path_buf(), + }); + } + validate_selection_with_absence(engine, receipt, &selection, None, Some(witness))?; + Ok(selection) +} + +/// Selection binds only this run's assignments. Independent sibling +/// allocations may advance the global serial without changing its authority. +pub(crate) fn scoped_absence_projection( + candidate: &Candidate, + engine: &Engine<'_>, + receipt: &Receipt, + previous_boot: &str, +) -> Result { + let store = strict_store(candidate, engine)?; + let selected = store + .slots + .iter() + .filter(|(_, assignment)| assignment.run == receipt.run) + .collect::>(); + Ok(json!({ + "owner": engine.guest().incarnation(), + "current_boot": engine.guest().boot_id(), + "previous_boot": previous_boot, + "run": receipt.run, + "plan": receipt.plan_id, + "capacity": engine.guest().bridge_intent().map_or(0, |value| value.slots), + "selected": selected, + })) +} + +pub(crate) fn selection_absence_projection(selection: &Selection) -> Result { + Ok(json!({ + "owner": selection.owner, + "current_boot": selection.boot, + "previous_boot": selection.previous_boot, + "run": selection.run, + "plan": selection.plan, + "capacity": selection.capacity, + "selected": selection.selected.iter() + .map(|(slot, selected)| (slot, &selected.assignment)) + .collect::>(), + })) +} + +pub(crate) fn verify_remaining_absence( + candidate: &Candidate, + engine: &Engine<'_>, + receipt: &Receipt, + selection: &Selection, + witness: &super::super::absent_publication_cleanup::Selection, +) -> Result<(), CandidateError> { + validate_selection_with_absence(engine, receipt, selection, None, Some(witness))?; + let store = strict_store(candidate, engine)?; + remaining_matches(&store, receipt, selection) +} + /// Pending recovery may resume only the exact remaining reservations. Released /// slots may be absent; a replacement or new reservation is never adopted. pub(crate) fn verify_remaining_recovery( @@ -344,6 +502,7 @@ pub(crate) fn verify_live_remaining( fn live_remaining_matches(store: &Store, selection: &Selection) -> Result<(), CandidateError> { if selection.previous_boot.is_some() || selection.predecessor_owner.is_some() + || selection.predecessor_absence.is_some() || store.owner != selection.owner || store.next_launch_serial < selection.serial { @@ -414,6 +573,7 @@ fn prior_generation( && prior.boot == current.previous_boot.as_deref().unwrap_or_default() && prior.previous_boot.is_none() && prior.predecessor_owner.is_none() + && prior.predecessor_absence.is_none() && prior.run == current.run && prior.plan == current.plan && prior.capacity == current.capacity @@ -436,6 +596,7 @@ fn prior_generation( && prior.boot == current.previous_boot.as_deref().unwrap_or_default() && prior.previous_boot.is_none() && prior.predecessor_owner.is_none() + && prior.predecessor_absence.is_none() && prior.run == current.run && prior.plan == current.plan && prior.capacity == current.capacity @@ -575,6 +736,16 @@ pub(crate) fn read_recovery( validate_selection_recovery(engine, receipt, &selection, host_pin)?; Ok(selection) } +pub(crate) fn read_recovery_absence( + engine: &Engine<'_>, + receipt: &Receipt, + root: &std::path::Path, + witness: &super::super::absent_publication_cleanup::Selection, +) -> Result { + let selection = state::read_bounded(&selection_path(root)?, 65536)?; + validate_selection_with_absence(engine, receipt, &selection, None, Some(witness))?; + Ok(selection) +} pub(crate) fn verify_recovery( candidate: &Candidate, @@ -598,6 +769,30 @@ pub(crate) fn verify_recovery( engine.guest().verify()?; Ok(()) } +pub(crate) fn verify_recovery_absence( + candidate: &Candidate, + engine: &Engine<'_>, + receipt: &Receipt, + selection: &Selection, + witness: &super::super::absent_publication_cleanup::Selection, +) -> Result<(), CandidateError> { + validate_selection_with_absence(engine, receipt, selection, None, Some(witness))?; + let store = strict_store(candidate, engine)?; + if store.next_launch_serial < selection.serial + || store + .slots + .values() + .any(|assignment| assignment.run == receipt.run) + { + return Err(invalid()); + } + for (slot, selected) in &selection.selected { + if let Some(helper) = &selected.helper { + relay::verify_cleanup(engine, *slot, &selected.assignment, helper)?; + } + } + engine.guest().verify() +} #[cfg(test)] pub(crate) fn verify( @@ -622,6 +817,7 @@ mod tests { boot: "boot".into(), previous_boot: None, predecessor_owner: None, + predecessor_absence: None, run: "b".repeat(32), plan: "c".repeat(64), capacity: 0, diff --git a/packages/runtime-core/src/provider/graph/cleanup_enrollment.rs b/packages/runtime-core/src/provider/graph/cleanup_enrollment.rs index 6c187f2e0..665db1b7b 100644 --- a/packages/runtime-core/src/provider/graph/cleanup_enrollment.rs +++ b/packages/runtime-core/src/provider/graph/cleanup_enrollment.rs @@ -119,6 +119,7 @@ pub(super) fn retention(root: &Path, receipt: &Receipt) -> Result<(), CandidateE #[cfg(target_os = "macos")] if super::live_owner_cleanup::retained(root, receipt)? || super::dead_owner_cleanup::retained(root, receipt)? + || super::absent_publication_cleanup::retained(root, receipt)? { return Ok(()); } diff --git a/packages/runtime-core/src/provider/graph/dead_owner_cleanup.rs b/packages/runtime-core/src/provider/graph/dead_owner_cleanup.rs index 9e32a865f..14d5599b4 100644 --- a/packages/runtime-core/src/provider/graph/dead_owner_cleanup.rs +++ b/packages/runtime-core/src/provider/graph/dead_owner_cleanup.rs @@ -38,7 +38,7 @@ fn selected(receipt: &Receipt) -> Result { &serde_json::to_vec_pretty(receipt).map_err(|_| refused())?, )) } -fn immutable(receipt: &Receipt) -> Result { +pub(super) fn immutable(receipt: &Receipt) -> Result { let mut value = serde_json::to_value(receipt).map_err(|_| refused())?; value.as_object_mut().ok_or_else(refused)?.remove("phase"); for group in ["resources", "probes"] { diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test.rs b/packages/runtime-core/src/provider/graph/foreground/native_test.rs index aba7ac312..84665fc9e 100644 --- a/packages/runtime-core/src/provider/graph/foreground/native_test.rs +++ b/packages/runtime-core/src/provider/graph/foreground/native_test.rs @@ -542,6 +542,7 @@ mod dependency_rebind; mod dependency_slots; mod startup_cancellation; +mod absent_publication_recovery; mod host_pin_recovery; mod retired_recovery_cleanup; mod same_boot_recovery; diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs b/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs new file mode 100644 index 000000000..aa290d1d8 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs @@ -0,0 +1,500 @@ +//! Owned capacity-two VM qualification for absent post-host-reboot publication. +//! The host reboot evidence is explicitly simulated in this isolated fixture; +//! this does not prove physical-volume continuity on a real reboot. +use super::dependency_rebind::{checked_cli, exec, snapshot}; +use super::*; +use crate::provider::{lifecycle, state::Owner}; +use std::os::unix::fs::MetadataExt; + +fn ready(owner: &mut Process, run: &str, deadline: Instant) { + loop { + assert!( + owner.poll().is_none(), + "foreground owner exited before ready" + ); + if let Some(end) = owner.out.iter().position(|byte| *byte == b'\n') { + let value: Value = serde_json::from_slice(&owner.out[..end]).unwrap(); + assert_eq!(value["kind"], "graph_foreground_ready"); + assert_eq!(value["run"], run); + return; + } + assert!(Instant::now() < deadline, "foreground readiness deadline"); + std::thread::sleep(Duration::from_millis(10)); + } +} + +fn refused_cli( + binary: &Path, + candidate: &Candidate, + args: &[&str], + expected_code: &str, + deadline: Instant, +) { + let mut process = Process::start(binary, candidate, args, None); + let status = process.wait(deadline); + assert_eq!( + status.code(), + Some(2), + "expected a CLI refusal, not a crash" + ); + assert!(process.out.is_empty(), "refusal emitted a result"); + let error: Value = serde_json::from_slice(&process.err).expect("structured CLI error"); + assert_eq!(error["code"], expected_code); +} + +fn inspect_args<'a>(run: &'a str, old: &'a str, prior: &'a str) -> [&'a str; 9] { + [ + "graph", + "inspect-absent-publication-cleanup", + "--run-id", + run, + "--original-owner-file", + old, + "--host-inspection-file", + prior, + "--json", + ] +} +fn recover_args<'a>(run: &'a str, old: &'a str, prior: &'a str, hash: &'a str) -> [&'a str; 13] { + [ + "graph", + "recover-absent-publication-cleanup", + "--run-id", + run, + "--original-owner-file", + old, + "--host-inspection-file", + prior, + "--expect-selection", + hash, + "--retain-data", + "--accept-unpinned-post-reboot", + "--json", + ] +} + +/// Scoped fixture file mutation restores only bytes it wrote, including on a +/// panic. It never replaces managed provider or graph state. +struct PrivateBytes<'a> { + path: &'a Path, + original: Vec, + substituted: Vec, +} +impl<'a> PrivateBytes<'a> { + fn replace(path: &'a Path, substituted: Vec) -> Self { + let original = fs::read(path).unwrap(); + fs::write(path, &substituted).unwrap(); + Self { + path, + original, + substituted, + } + } +} +impl Drop for PrivateBytes<'_> { + fn drop(&mut self) { + if fs::read(self.path).ok().as_deref() == Some(self.substituted.as_slice()) { + fs::write(self.path, &self.original).unwrap(); + } + } +} + +struct Moved { + original: PathBuf, + moved: PathBuf, +} +impl Moved { + fn new(original: PathBuf) -> Self { + let moved = original.with_extension("absent-fixture-held"); + assert!(!moved.exists()); + fs::rename(&original, &moved).unwrap(); + Self { original, moved } + } +} +impl Drop for Moved { + fn drop(&mut self) { + if self.moved.exists() { + assert!( + !self.original.exists(), + "foreign path replaced selected fixture" + ); + fs::rename(&self.moved, &self.original).unwrap(); + } + } +} + +struct FaultChild(Option); +impl Drop for FaultChild { + fn drop(&mut self) { + if let Some(mut child) = self.0.take() { + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +#[test] +#[ignore = "Only the parent owned VM fixture starts this exact fault helper"] +fn absent_publication_after_bridge_fault_child() { + let candidate = + Candidate::discover(Path::new(&std::env::var("HACK_LOCAL_TEST_ROOT").unwrap())).unwrap(); + graph::absent_publication_cleanup::recover( + &candidate, + &std::env::var("HACK_LOCAL_GRAPH_RUN").unwrap(), + &std::env::var("HACK_LOCAL_GRAPH_SELECTION").unwrap(), + Path::new(&std::env::var("HACK_LOCAL_GRAPH_OLD_OWNER").unwrap()), + Path::new(&std::env::var("HACK_LOCAL_GRAPH_INSPECTION").unwrap()), + ) + .unwrap(); +} + +#[test] +#[ignore = "Owned capacity-two native VM, pinned image and external 300s watchdog required"] +fn explicit_absent_publication_cleanup_retains_selected_volume_and_sibling() { + let deadline = Instant::now() + Duration::from_secs(270); + let candidate = + Candidate::discover(Path::new(&std::env::var("HACK_LOCAL_TEST_ROOT").unwrap())).unwrap(); + let binary = PathBuf::from(std::env::var("HACK_LOCAL_TEST_BINARY").unwrap()); + let image = std::env::var("HACK_LOCAL_TEST_IMAGE").unwrap(); + let fixtures = [graph::tests::Fixture::new(), graph::tests::Fixture::new()]; + let runs = [ + graph::probes::token().unwrap(), + graph::probes::token().unwrap(), + ]; + let private = graph::tests::Fixture::new(); + let mut plans = Vec::new(); + let mut selections = Vec::new(); + let mut dependencies = Vec::new(); + for (index, fixture) in fixtures.iter().enumerate() { + state::write(&fixture.0.join("compose.yaml"), &json!({ + "services":{"web":{"image":image,"read_only":true,"network_mode":"none", + "init":true,"user":"0:0","entrypoint":["/bin/sleep","300"],"command":[], + "volumes":["data:/data"],"healthcheck":{"test":["CMD","/bin/hack-graph-startup-app","complete"], + "interval":"200ms","timeout":"2s","retries":10,"start_period":"500ms"}}}, + "volumes":{"data":{}} + })).unwrap(); + let review = project::plan( + &candidate, + project::PlanOptions { + branch: None, + project: &fixture.0, + compose_file: Path::new("compose.yaml"), + profiles: &[], + }, + ) + .unwrap(); + let selection = private.0.join(format!("dependencies-{index}.json")); + state::write(&selection, &json!({"version":1,"plan":review.plan_id, + "artifact":"/tmp/unused-control-only-artifact","artifact_sha256":"a".repeat(64),"dependencies":[]})).unwrap(); + let dependency = checked_cli( + "dependency-plan", + &binary, + &candidate, + &[ + "graph", + "dependency-plan", + "--dependencies", + selection.to_str().unwrap(), + "--json", + ], + deadline, + ); + plans.push(review); + selections.push(selection); + dependencies.push(dependency); + } + let start = |index: usize| { + let normalized = fixtures[index].0.join("compose.yaml"); + Process::start( + &binary, + &candidate, + &[ + "graph", + "serve", + "--project", + fixtures[index].0.to_str().unwrap(), + "--file", + "compose.yaml", + "--expect-plan", + &plans[index].plan_id, + "--run-id", + &runs[index], + "--ready", + "web=healthy", + "--timeout-seconds", + "90", + "--dependencies", + selections[index].to_str().unwrap(), + "--expect-dependencies", + dependencies[index]["dependency_plan_id"].as_str().unwrap(), + "--normalized-file", + normalized.to_str().unwrap(), + "--expect-original", + &plans[index].plan.compose_sha256, + "--expect-namespace", + &plans[index].plan.namespace, + "--json", + ], + None, + ) + }; + let mut selected_owner = start(0); + let mut selected_cleanup = Cleanup { + binary: &binary, + candidate: &candidate, + run: &runs[0], + done: false, + }; + ready(&mut selected_owner, &runs[0], deadline); + assert_eq!( + exec(&binary, &candidate, &runs[0], "web", "write-data", deadline)["exit_code"], + 0 + ); + let original = snapshot(&candidate, &runs[0], deadline).receipt; + assert_eq!(original.phase, "ready-observed"); + let old_owner = Owner::load(&candidate).unwrap(); + let old_boot = old_owner.guest_boot_id.clone().unwrap(); + selected_owner.child.kill().unwrap(); + selected_owner.wait(deadline); + assert_eq!(lifecycle::down(&candidate).unwrap().phase, "stopped"); + let new_boot = lifecycle::up(&candidate).unwrap().guest_boot_id.unwrap(); + assert_ne!(old_boot, new_boot); + + let mut sibling_owner = start(1); + let mut sibling_cleanup = Cleanup { + binary: &binary, + candidate: &candidate, + run: &runs[1], + done: false, + }; + ready(&mut sibling_owner, &runs[1], deadline); + assert_eq!( + exec(&binary, &candidate, &runs[1], "web", "write-data", deadline)["exit_code"], + 0 + ); + let sibling = serde_json::to_vec(&snapshot(&candidate, &runs[1], deadline).receipt).unwrap(); + + let current = Owner::load(&candidate).unwrap(); + let boot = lifecycle::host_filesystem::host_boot_micros().unwrap(); + let new_device = current.storage.as_ref().unwrap().device; + let old_device = new_device.checked_add(1).unwrap(); + let mut synthetic = old_owner.clone(); + synthetic.storage.as_mut().unwrap().device = old_device; + synthetic.overlay.as_mut().unwrap().device = old_device; + if let Some(share) = synthetic.project_share.as_mut() { + share.device = old_device; + } + let prior_process = synthetic.process.as_mut().unwrap(); + prior_process.pid = i32::MAX; + prior_process.start_micros = boot - 1; + let old_path = private.0.join("pre-host-boot-owner.json"); + state::write(&old_path, &synthetic).unwrap(); + let old_bytes = fs::read(&old_path).unwrap(); + let pool = fs::symlink_metadata(candidate.state_root.join("run/smolvm")).unwrap(); + let inspection_path = private.0.join("pre-migration-inspection.json"); + state::write( + &inspection_path, + &serde_json::from_slice::(&graph::absent_publication_cleanup::fixture_inspection( + &old_bytes, + ¤t, + old_device, + boot, + pool.ino(), + )) + .unwrap(), + ) + .unwrap(); + let graph_root = graph::directory(&candidate, &runs[0]).unwrap(); + let raw_graph = fs::read(graph_root.join("state.json")).unwrap(); + let publisher_root = transport::root(&candidate, &runs[0]).unwrap(); + let control_root = original + .relay_startup + .as_ref() + .unwrap() + .control_root + .clone(); + assert!(publisher_root.exists() && control_root.exists()); + fs::remove_dir_all(&publisher_root).unwrap(); + fs::remove_dir_all(&control_root).unwrap(); + + let old = old_path.to_str().unwrap(); + let prior = inspection_path.to_str().unwrap(); + let initial = checked_cli( + "inspect-absent-publications", + &binary, + &candidate, + &inspect_args(&runs[0], old, prior), + deadline, + ); + let selection = initial["selection_sha256"].as_str().unwrap(); + assert_eq!(selection.len(), 64); + let intent_path = graph_root.join("absent-publication-cleanup.json"); + refused_cli( + &binary, + &candidate, + &recover_args(&runs[0], old, prior, &"0".repeat(64)), + "graph_absent_publication_recovery", + deadline, + ); + assert!(!intent_path.exists()); + assert_eq!(fs::read(graph_root.join("state.json")).unwrap(), raw_graph); + let refreshed = checked_cli( + "inspect-after-stale-selection", + &binary, + &candidate, + &inspect_args(&runs[0], old, prior), + deadline, + ); + assert_eq!(refreshed["selection_sha256"], selection); + { + let mut whitespace = old_bytes.clone(); + whitespace.push(b' '); + let _changed = PrivateBytes::replace(&old_path, whitespace); + refused_cli( + &binary, + &candidate, + &recover_args(&runs[0], old, prior, selection), + "graph_absent_publication_recovery", + deadline, + ); + assert!(!intent_path.exists()); + } + { + let _missing = Moved::new(inspection_path.clone()); + refused_cli( + &binary, + &candidate, + &recover_args(&runs[0], old, prior, selection), + "graph_absent_publication_recovery", + deadline, + ); + assert!(!intent_path.exists()); + } + { + let state_path = graph_root.join("state.json"); + let mut changed = raw_graph.clone(); + changed.push(b' '); + let _changed = PrivateBytes::replace(&state_path, changed); + refused_cli( + &binary, + &candidate, + &recover_args(&runs[0], old, prior, selection), + "graph_absent_publication_recovery", + deadline, + ); + assert!(!intent_path.exists()); + } + { + let lock = publisher_root.join("operation.lock"); + let _original = Moved::new(lock.clone()); + fs::write(&lock, b"substituted lock path").unwrap(); + refused_cli( + &binary, + &candidate, + &recover_args(&runs[0], old, prior, selection), + "graph_absent_publication_recovery", + deadline, + ); + assert!(!intent_path.exists()); + fs::remove_file(&lock).unwrap(); + } + let child = Command::new(std::env::current_exe().unwrap()) + .args(["--ignored", "--exact", + "provider::graph::foreground::native_test::absent_publication_recovery::absent_publication_after_bridge_fault_child"]) + .env("HACK_LOCAL_TEST_ROOT", std::env::var("HACK_LOCAL_TEST_ROOT").unwrap()) + .env("HACK_LOCAL_GRAPH_RUN", &runs[0]) + .env("HACK_LOCAL_GRAPH_SELECTION", selection) + .env("HACK_LOCAL_GRAPH_OLD_OWNER", old) + .env("HACK_LOCAL_GRAPH_INSPECTION", prior) + .env("HACK_LOCAL_GRAPH_FAULT", "absent-after-bridge-release") + .stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()) + .spawn().unwrap(); + let mut fault = FaultChild(Some(child)); + let marker = graph_root.join("fault-absent-after-bridge-release.json"); + let fault_deadline = Instant::now() + Duration::from_secs(25); + while !marker.exists() { + assert!( + fault.0.as_mut().unwrap().try_wait().unwrap().is_none(), + "fault child exited before selected boundary" + ); + assert!( + Instant::now() < fault_deadline, + "selected fault boundary deadline" + ); + std::thread::sleep(Duration::from_millis(20)); + } + assert_eq!(state::read::(&marker).unwrap()["run"], runs[0]); + assert!( + intent_path.exists(), + "absence intent was durable before bridge release" + ); + assert_eq!( + state::read::(&graph_root.join("state.json")) + .unwrap() + .phase, + "cleanup-intent", + "graph intent was durable before bridge release" + ); + assert!( + graph::absent_publication_cleanup::publication_allowed(&candidate, &runs[0], false) + .is_err() + ); + assert!( + graph::absent_publication_cleanup::publication_allowed(&candidate, &runs[0], true).is_err() + ); + fault.0.as_mut().unwrap().kill().unwrap(); + fault.0.as_mut().unwrap().wait().unwrap(); + fault.0 = None; + let result = checked_cli( + "recover-absent-publications", + &binary, + &candidate, + &recover_args(&runs[0], old, prior, selection), + deadline, + ); + assert_eq!(result["phase"], "stopped-data-retained"); + assert_eq!(result["publisher_retired"], true); + let stopped = snapshot(&candidate, &runs[0], deadline).receipt; + assert_eq!( + stopped.resources["volume:data"].name, + original.resources["volume:data"].name + ); + let repeated = checked_cli( + "idempotent-absent-retirement", + &binary, + &candidate, + &recover_args(&runs[0], old, prior, selection), + deadline, + ); + assert_eq!(repeated["publisher_retired"], true); + assert!(sibling_owner.poll().is_none()); + assert_eq!( + serde_json::to_vec(&snapshot(&candidate, &runs[1], deadline).receipt).unwrap(), + sibling + ); + assert_eq!( + exec(&binary, &candidate, &runs[1], "web", "read-data", deadline)["exit_code"], + 0 + ); + + // Keep the selected retained volume for the later source-continuity + // fixture. The external harness tears down this isolated test pool. + selected_cleanup.done = true; + let removed = checked_cli( + "remove-sibling-data", + &binary, + &candidate, + &[ + "graph", + "cleanup", + "--run-id", + &runs[1], + "--remove-data", + "--json", + ], + deadline, + ); + assert_eq!(removed["phase"], "removed"); + assert!(sibling_owner.wait(deadline).success()); + sibling_cleanup.done = true; +} diff --git a/packages/runtime-core/src/provider/graph/foreground/transport.rs b/packages/runtime-core/src/provider/graph/foreground/transport.rs index 2064bfbc6..975cb8540 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport.rs @@ -772,6 +772,7 @@ impl Publication { Self::bind_mode(candidate, run, true) } fn bind_mode(candidate: &Candidate, run: &str, retired: bool) -> Result { + super::super::absent_publication_cleanup::publication_allowed(candidate, run, retired)?; let root = root(candidate, run)?; let lock = if retired { state::check_private_directory(&root).map_err(|_| refused())?; @@ -780,6 +781,8 @@ impl Publication { state::private_directory(&root).map_err(|_| refused())?; state::Lock::acquire(&root).map_err(|_| refused())? }; + // The foreground lock serializes a concurrent absence-intent writer. + super::super::absent_publication_cleanup::publication_allowed(candidate, run, retired)?; for name in ["control.sock", "owner.json"] { match fs::symlink_metadata(root.join(name)) { Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} diff --git a/packages/runtime-core/src/provider/graph/host_pin_recovery.rs b/packages/runtime-core/src/provider/graph/host_pin_recovery.rs index a41c94b31..1846c7018 100644 --- a/packages/runtime-core/src/provider/graph/host_pin_recovery.rs +++ b/packages/runtime-core/src/provider/graph/host_pin_recovery.rs @@ -40,7 +40,7 @@ fn digest(bytes: &[u8]) -> String { /// guest check so a changed receipt cannot impersonate the running kernel boot. /// execute_cleanup itself compares /proc boot ID and /storage owner under the /// held Engine/Guest lease before this fixed acknowledgement is emitted. -fn verify_guest_identity(engine: &Engine<'_>) -> Result<(), CandidateError> { +pub(super) fn verify_guest_identity(engine: &Engine<'_>) -> Result<(), CandidateError> { let acknowledged = engine .guest() .execute_cleanup("printf 'host-pin-guest-identity-v1\\n'", &[]) @@ -50,7 +50,7 @@ fn verify_guest_identity(engine: &Engine<'_>) -> Result<(), CandidateError> { } Ok(()) } -fn read_raw(path: &Path, limit: u64) -> Result, CandidateError> { +pub(super) fn read_raw(path: &Path, limit: u64) -> Result, CandidateError> { read_raw_with(path, limit, || {}) } fn read_raw_with( @@ -114,7 +114,7 @@ fn absent(path: &Path) -> Result { } } -fn exact_lock_path(root: &Path, held: &state::Lock) -> Result<(), CandidateError> { +pub(super) fn exact_lock_path(root: &Path, held: &state::Lock) -> Result<(), CandidateError> { let pathname = fs::symlink_metadata(root.join("operation.lock")).map_err(|_| refused())?; if !pathname.is_file() || pathname.nlink() != 1 @@ -258,7 +258,7 @@ pub(super) fn selected_for_old_publisher( Ok(Some(witness)) } -fn verify_volume_projections( +pub(super) fn verify_volume_projections( engine: &Engine<'_>, receipt: &Receipt, selected: &BTreeMap, diff --git a/packages/runtime-core/src/provider/graph/host_relay.rs b/packages/runtime-core/src/provider/graph/host_relay.rs index 49946e7f5..967b8076e 100644 --- a/packages/runtime-core/src/provider/graph/host_relay.rs +++ b/packages/runtime-core/src/provider/graph/host_relay.rs @@ -508,6 +508,47 @@ pub(super) fn inspect_cleanup_recovery( environment: &super::super::environment_recovery::GraphInventory, bridge_selection: &bridges::cleanup::Selection, host_pin: Option<&super::host_pin_recovery::Witness>, +) -> Result<[u8; 32], CandidateError> { + inspect_cleanup_selected( + candidate, + engine, + expected, + remove_data, + environment, + bridge_selection, + SelectedCleanupProof::Existing(host_pin), + ) +} +pub(super) fn inspect_cleanup_absence( + candidate: &Candidate, + engine: &Engine<'_>, + expected: &Receipt, + environment: &super::super::environment_recovery::GraphInventory, + bridge_selection: &bridges::cleanup::Selection, + witness: &super::absent_publication_cleanup::Selection, +) -> Result<[u8; 32], CandidateError> { + inspect_cleanup_selected( + candidate, + engine, + expected, + false, + environment, + bridge_selection, + SelectedCleanupProof::Absence(witness), + ) +} +enum SelectedCleanupProof<'a> { + Existing(Option<&'a super::host_pin_recovery::Witness>), + Absence(&'a super::absent_publication_cleanup::Selection), +} +fn inspect_cleanup_selected( + candidate: &Candidate, + engine: &Engine<'_>, + expected: &Receipt, + remove_data: bool, + environment: &super::super::environment_recovery::GraphInventory, + bridge_selection: &bridges::cleanup::Selection, + proof: SelectedCleanupProof<'_>, ) -> Result<[u8; 32], CandidateError> { let (receipt, root) = archive::load_confirmation(candidate, engine, &expected.run)?; match fs::symlink_metadata(root.join("state.pending")) { @@ -552,10 +593,33 @@ pub(super) fn inspect_cleanup_recovery( require_retained_volume(resource, value.is_some(), remove_data)?; observations.insert(key, value); } - if &bridges::cleanup::read_recovery(engine, &receipt, &root, host_pin)? != bridge_selection { + let recorded = match proof { + SelectedCleanupProof::Absence(witness) => { + bridges::cleanup::read_recovery_absence(engine, &receipt, &root, witness)? + } + SelectedCleanupProof::Existing(host_pin) => { + bridges::cleanup::read_recovery(engine, &receipt, &root, host_pin)? + } + }; + if &recorded != bridge_selection { return Err(refused()); } - bridges::cleanup::verify_recovery(candidate, engine, &receipt, bridge_selection, host_pin)?; + match proof { + SelectedCleanupProof::Absence(witness) => bridges::cleanup::verify_recovery_absence( + candidate, + engine, + &receipt, + bridge_selection, + witness, + )?, + SelectedCleanupProof::Existing(host_pin) => bridges::cleanup::verify_recovery( + candidate, + engine, + &receipt, + bridge_selection, + host_pin, + )?, + } environment::verify_cleanup(candidate, engine, &receipt, &root, environment)?; probes::verify_cleanup(engine, &receipt)?; startup::verify_cleanup(engine, &receipt)?; diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index d43122f1e..9b17ba11b 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -12,6 +12,8 @@ pub use normalized::run_normalized_with_host_dependencies_until; pub use normalized::{ NormalizedInputIdentity, NormalizedRunOptions, compile_normalized_inputs, run_normalized, }; +#[cfg(target_os = "macos")] +mod absent_publication_cleanup; mod admission; mod cache_provenance; mod dependency_hosts; @@ -20,6 +22,10 @@ mod dependency_slots; #[cfg(target_os = "macos")] mod host_pin_recovery; #[cfg(target_os = "macos")] +pub use absent_publication_cleanup::{ + inspect as inspect_absent_publication_cleanup, recover as recover_absent_publication_cleanup, +}; +#[cfg(target_os = "macos")] pub use dependency_slots::{ inspect as dependency_reservations, recover_orphan as recover_dependency_reservation, }; @@ -1567,11 +1573,28 @@ pub fn cleanup( // The caller retains the VM mutation lease through this entire effect and any // additional relay confirmation. Never reconnect from inside this function. fn cleanup_owned( + candidate: &Candidate, + engine: &Engine<'_>, + receipt: Receipt, + root: &std::path::Path, + remove_data: bool, +) -> Result { + cleanup_owned_fenced(candidate, engine, receipt, root, remove_data, false, || { + Ok(()) + }) +} + +/// An explicit recovery can recheck its independent host proof immediately +/// before each destructive graph effect. Ordinary callers use the same engine +/// lease and state machine without an additional recovery witness. +fn cleanup_owned_fenced( candidate: &Candidate, engine: &Engine<'_>, mut receipt: Receipt, root: &std::path::Path, remove_data: bool, + early_intent: bool, + fence: impl Fn() -> Result<(), CandidateError>, ) -> Result { if root.join("state.pending").exists() || root.join("state.pending").is_symlink() { return Err(error( @@ -1579,11 +1602,35 @@ fn cleanup_owned( "Pending graph journal is retained; cleanup is blocked until journal reconciliation.", )); } + fence()?; + // Absence recovery must commit the graph cleanup intent before changing a + // selected relay/bridge inventory. A crash can then resume from an exact + // durable intent even when an early external effect already completed. + let environment_slots = if early_intent { + let slots = environment::cleanup_slots(candidate, engine, &receipt)?; + receipt.phase = "cleanup-intent".into(); + state::write(&root.join("state.json"), &receipt)?; + fence()?; + slots + } else { + Vec::new() + }; startup::cleanup_guest(engine, &receipt, false)?; + fence()?; bridges::release_run(candidate, engine, &receipt)?; - let environment_slots = environment::cleanup_slots(candidate, engine, &receipt)?; - receipt.phase = "cleanup-intent".into(); - state::write(&root.join("state.json"), &receipt)?; + #[cfg(test)] + if early_intent { + fault_pause(root, &receipt.run, "absent-after-bridge-release")?; + } + let environment_slots = if early_intent { + environment_slots + } else { + let slots = environment::cleanup_slots(candidate, engine, &receipt)?; + receipt.phase = "cleanup-intent".into(); + state::write(&root.join("state.json"), &receipt)?; + slots + }; + fence()?; shutdown::stop_owned(engine, &receipt, root)?; for kind in [Kind::Container, Kind::Network, Kind::Volume] { if kind == Kind::Volume && !remove_data { @@ -1605,12 +1652,14 @@ fn cleanup_owned( state::write(&root.join("state.json"), &receipt)?; continue; } + fence()?; if let Some(value) = inspect_resource(engine, &receipt, &resource)? { let target = if kind == Kind::Volume { resource.name.as_str() } else { value["Id"].as_str().expect("verified id") }; + fence()?; engine.request( Method::DELETE, &format!( @@ -1639,12 +1688,15 @@ fn cleanup_owned( fault_pause(root, &receipt.run, "cleanup-after-remove")?; } } + fence()?; startup::cleanup_guest(engine, &receipt, true)?; startup::verify_cleanup(engine, &receipt)?; probes::cleanup(engine, &mut receipt)?; for slot in environment_slots { + fence()?; super::environment_recovery::retire(candidate, engine.guest(), &slot, None)?; } + fence()?; receipt.phase = if remove_data { "removed" } else { @@ -1652,6 +1704,7 @@ fn cleanup_owned( } .into(); state::write(&root.join("state.json"), &receipt)?; + fence()?; Ok(receipt) } diff --git a/packages/runtime-core/tests/absent_recovery_cli_contract.rs b/packages/runtime-core/tests/absent_recovery_cli_contract.rs new file mode 100644 index 000000000..3ea54549d --- /dev/null +++ b/packages/runtime-core/tests/absent_recovery_cli_contract.rs @@ -0,0 +1,107 @@ +//! Recovery acknowledgements must reject at the CLI boundary before any runtime admission. +use serde_json::Value; +use std::{ + fs, + path::{Path, PathBuf}, + process::Command, +}; + +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + let root = std::env::temp_dir().join(format!( + "hack-absence-cli-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&root).unwrap(); + Self(root.canonicalize().unwrap()) + } +} +impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_dir_all(&self.0).unwrap(); + } +} + +#[test] +fn absent_recovery_requires_one_exact_selection_and_both_acknowledgements() { + let fixture = Fixture::new(); + // The development executable intentionally resolves its compiled checkout + // before command dispatch. Supply that valid identity so these assertions + // exercise recovery parsing instead of the unrelated checkout gate. + let checkout = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .unwrap(); + let original = fixture.0.join("original-not-created.json"); + let inspection = fixture.0.join("inspection-not-created.json"); + let complete: Vec = [ + "graph", + "recover-absent-publication-cleanup", + "--run-id", + &"a".repeat(32), + "--original-owner-file", + original.to_str().unwrap(), + "--host-inspection-file", + inspection.to_str().unwrap(), + "--expect-selection", + &"b".repeat(64), + "--retain-data", + "--accept-unpinned-post-reboot", + "--json", + ] + .into_iter() + .map(str::to_owned) + .collect(); + let mut cases = Vec::new(); + for flag in ["--retain-data", "--accept-unpinned-post-reboot"] { + let mut missing = complete.clone(); + missing.retain(|argument| argument != flag); + cases.push(missing); + let mut duplicated = complete.clone(); + duplicated.push(flag.into()); + cases.push(duplicated); + } + for flag in [ + "--run-id", + "--original-owner-file", + "--host-inspection-file", + "--expect-selection", + ] { + let mut missing = complete.clone(); + let position = missing + .iter() + .position(|argument| argument == flag) + .unwrap(); + missing.drain(position..position + 2); + cases.push(missing); + } + for extra in ["--remove-data", "--json", "--force"] { + let mut invalid = complete.clone(); + invalid.push(extra.into()); + cases.push(invalid); + } + for arguments in cases { + let output = Command::new(env!("CARGO_BIN_EXE_hack-runtime-candidate")) + .arg("--candidate-root") + .arg(&checkout) + .args(&arguments) + .env_clear() + .env("PATH", "/nonexistent") + .env("HOME", fixture.0.join("home-not-created")) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2), "{arguments:?}"); + assert!( + output.stdout.is_empty(), + "refusal must not emit a success result" + ); + let failure: Value = serde_json::from_slice(&output.stderr).unwrap(); + assert_eq!(failure["code"], "graph_arguments", "{arguments:?}"); + assert_eq!(fs::read_dir(&fixture.0).unwrap().count(), 0); + } +} From 84ba81fa12ece32864b677475494061b8abc7c9c Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 03:40:44 -0400 Subject: [PATCH 7/9] fix(runtime): release recovery VM locks with inherited descriptors --- .../runtime-core/src/provider/lifecycle.rs | 24 ++++++++-- .../provider/lifecycle/short_home/tests.rs | 48 +++++++++++++++++++ 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/packages/runtime-core/src/provider/lifecycle.rs b/packages/runtime-core/src/provider/lifecycle.rs index 36decce95..11eba0fdd 100644 --- a/packages/runtime-core/src/provider/lifecycle.rs +++ b/packages/runtime-core/src/provider/lifecycle.rs @@ -1818,8 +1818,25 @@ fn finish_absent( finish_absent_locked(candidate, owner, value, record_disks, &vm_lock) } -/// Keep this descriptor through alias restoration and the stopped receipt. -fn lock_absent_disks(candidate: &Candidate, owner: &Owner) -> Result { +/// Closing alone can leave a flock held by a fork/dup copy until that copy closes. +/// Explicitly unlock this open-file description when the recovery scope ends. +struct VmLock(File); +impl std::ops::Deref for VmLock { + type Target = File; + fn deref(&self) -> &File { + &self.0 + } +} +impl Drop for VmLock { + fn drop(&mut self) { + use std::os::fd::AsRawFd; + // SAFETY: the owned descriptor remains live through this Drop call. + unsafe { libc::flock(self.0.as_raw_fd(), libc::LOCK_UN) }; + } +} + +/// Keep this guard through alias restoration and the stopped receipt. +fn lock_absent_disks(candidate: &Candidate, owner: &Owner) -> Result { let directory = owner.real_data_dir(candidate)?; let vm_lock = OpenOptions::new() .read(true) @@ -1833,13 +1850,14 @@ fn lock_absent_disks(candidate: &Candidate, owner: &Owner) -> Result Date: Wed, 30 Sep 2026 03:42:06 -0400 Subject: [PATCH 8/9] test(runtime): preserve private recovery inspection selection --- .../absent_publication_recovery.rs | 79 ++++++++++++++++++- 1 file changed, 75 insertions(+), 4 deletions(-) diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs b/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs index aa290d1d8..5fe2c3c39 100644 --- a/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs +++ b/packages/runtime-core/src/provider/graph/foreground/native_test/absent_publication_recovery.rs @@ -4,7 +4,10 @@ use super::dependency_rebind::{checked_cli, exec, snapshot}; use super::*; use crate::provider::{lifecycle, state::Owner}; -use std::os::unix::fs::MetadataExt; +use std::{ + io::Write, + os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt}, +}; fn ready(owner: &mut Process, run: &str, deadline: Instant) { loop { @@ -306,6 +309,8 @@ fn explicit_absent_publication_cleanup_retains_selected_volume_and_sibling() { .unwrap(); let graph_root = graph::directory(&candidate, &runs[0]).unwrap(); let raw_graph = fs::read(graph_root.join("state.json")).unwrap(); + let provider_owner_path = candidate.state_root.join("run/smolvm/owner.json"); + let raw_provider_owner = fs::read(&provider_owner_path).unwrap(); let publisher_root = transport::root(&candidate, &runs[0]).unwrap(); let control_root = original .relay_startup @@ -318,9 +323,61 @@ fn explicit_absent_publication_cleanup_retains_selected_volume_and_sibling() { fs::remove_dir_all(&control_root).unwrap(); let old = old_path.to_str().unwrap(); - let prior = inspection_path.to_str().unwrap(); + let original_inspection = inspection_path.to_str().unwrap(); + let original_selection = checked_cli( + "inspect-private-original-inspection", + &binary, + &candidate, + &inspect_args(&runs[0], old, original_inspection), + deadline, + ); + let original_selection_hash = original_selection["selection_sha256"].as_str().unwrap(); + assert_eq!(original_selection_hash.len(), 64); + let inspection_bytes = fs::read(&inspection_path).unwrap(); + fs::set_permissions(&inspection_path, fs::Permissions::from_mode(0o644)).unwrap(); + assert_eq!( + fs::symlink_metadata(&inspection_path).unwrap().mode() & 0o7777, + 0o644 + ); + refused_cli( + &binary, + &candidate, + &inspect_args(&runs[0], old, original_inspection), + "graph_absent_publication_recovery", + deadline, + ); + assert!( + !publisher_root.exists(), + "read-only refusal did not reserve a publisher" + ); + assert_eq!(fs::read(&provider_owner_path).unwrap(), raw_provider_owner); + assert_eq!(fs::read(graph_root.join("state.json")).unwrap(), raw_graph); + + // A new private path retains the exact inspected bytes, while its path is + // deliberately part of the fresh selection hash. The historical 0644 + // receipt remains untouched after the copy. + let private_inspection = private.0.join("private-inspection-copy.json"); + let mut copy = fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&private_inspection) + .unwrap(); + copy.write_all(&inspection_bytes).unwrap(); + copy.sync_all().unwrap(); + fs::File::open(&private.0).unwrap().sync_all().unwrap(); + assert_eq!(fs::read(&private_inspection).unwrap(), inspection_bytes); + assert_eq!( + fs::symlink_metadata(&private_inspection).unwrap().mode() & 0o7777, + 0o600 + ); + assert_eq!( + fs::symlink_metadata(&inspection_path).unwrap().mode() & 0o7777, + 0o644 + ); + let prior = private_inspection.to_str().unwrap(); let initial = checked_cli( - "inspect-absent-publications", + "inspect-exact-private-inspection-copy", &binary, &candidate, &inspect_args(&runs[0], old, prior), @@ -328,7 +385,21 @@ fn explicit_absent_publication_cleanup_retains_selected_volume_and_sibling() { ); let selection = initial["selection_sha256"].as_str().unwrap(); assert_eq!(selection.len(), 64); + assert_ne!( + selection, original_selection_hash, + "selection binds the new canonical path" + ); + assert_eq!(fs::read(&provider_owner_path).unwrap(), raw_provider_owner); + assert_eq!(fs::read(graph_root.join("state.json")).unwrap(), raw_graph); let intent_path = graph_root.join("absent-publication-cleanup.json"); + assert!(!intent_path.exists()); + refused_cli( + &binary, + &candidate, + &recover_args(&runs[0], old, prior, original_selection_hash), + "graph_absent_publication_recovery", + deadline, + ); refused_cli( &binary, &candidate, @@ -360,7 +431,7 @@ fn explicit_absent_publication_cleanup_retains_selected_volume_and_sibling() { assert!(!intent_path.exists()); } { - let _missing = Moved::new(inspection_path.clone()); + let _missing = Moved::new(private_inspection.clone()); refused_cli( &binary, &candidate, From 571862cc525a75af932ff3ce5ade684d78f08e8e Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Wed, 30 Sep 2026 03:47:09 -0400 Subject: [PATCH 9/9] test(runtime): isolate concurrent recovery fixture roots --- .../provider/lifecycle/short_home/tests.rs | 79 ++++++++++++++++--- 1 file changed, 67 insertions(+), 12 deletions(-) diff --git a/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs b/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs index d914962ea..2654463e9 100644 --- a/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs +++ b/packages/runtime-core/src/provider/lifecycle/short_home/tests.rs @@ -4,12 +4,71 @@ use std::{ fs, os::{ fd::AsRawFd, - unix::{fs::MetadataExt, net::UnixListener}, + unix::{ + fs::{DirBuilderExt, MetadataExt}, + net::UnixListener, + }, }, path::PathBuf, process::Command, + sync::atomic::{AtomicU64, Ordering}, }; +static NEXT_POOL: AtomicU64 = AtomicU64::new(0); + +fn pool_directory(timestamp: u128) -> PathBuf { + fs::canonicalize(std::env::temp_dir()) + .unwrap() + .join(format!( + "hack-home-recovery-{}-{timestamp}-{}", + std::process::id(), + NEXT_POOL.fetch_add(1, Ordering::Relaxed) + )) +} + +fn create_pool_directory(path: &std::path::Path) -> std::io::Result<()> { + fs::DirBuilder::new().mode(0o700).create(path) +} + +#[test] +fn parallel_pool_roots_are_unique_at_the_same_timestamp_and_never_adopt() { + let paths: Vec<_> = std::thread::scope(|scope| { + let workers: Vec<_> = (0..16) + .map(|_| { + scope.spawn(|| { + let path = pool_directory(123); + create_pool_directory(&path).unwrap(); + path + }) + }) + .collect(); + workers + .into_iter() + .map(|worker| worker.join().unwrap()) + .collect() + }); + assert_eq!( + paths + .iter() + .collect::>() + .len(), + 16 + ); + for path in paths { + let before = fs::symlink_metadata(&path).unwrap(); + assert_eq!(before.mode() & 0o777, 0o700); + fs::write(path.join("marker"), b"owned fixture").unwrap(); + assert_eq!( + create_pool_directory(&path).unwrap_err().kind(), + std::io::ErrorKind::AlreadyExists + ); + assert_eq!(fs::symlink_metadata(&path).unwrap().ino(), before.ino()); + assert_eq!(fs::read(path.join("marker")).unwrap(), b"owned fixture"); + fs::remove_file(path.join("marker")).unwrap(); + fs::remove_dir(path).unwrap(); + } +} + struct Pool { candidate: Candidate, owner: Owner, @@ -21,17 +80,13 @@ impl Pool { let mut process = identity::observe(child.id() as i32).unwrap(); child.kill().unwrap(); child.wait().unwrap(); - let directory = fs::canonicalize(std::env::temp_dir()) - .unwrap() - .join(format!( - "hack-home-recovery-{}-{}", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - state::private_directory(&directory).unwrap(); + let directory = pool_directory( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(), + ); + create_pool_directory(&directory).unwrap(); let candidate = Candidate::discover(&directory).unwrap(); let operation = state::Lock::acquire(&root(&candidate)).unwrap(); let mut owner = Owner::create(