From 8724c744456b6e4d47cf8139d15a798bcd291be2 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 12:33:28 -0400 Subject: [PATCH 01/47] feat(runtime): bind native foreground publication to host boot --- packages/runtime-core/README.md | 8 ++ .../provider/graph/native/foreground/owner.rs | 77 +++++++++++++++++-- .../provider/graph/native/foreground/tests.rs | 43 +++++++++++ 3 files changed, 123 insertions(+), 5 deletions(-) diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 43abe3538..9feba8153 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -111,6 +111,14 @@ this same subset. Its private `native-graph-owner` record and authenticated v2 `native-graph-control` status/cleanup protocol bind the native review, exact process incarnation, directory, socket, file and retained per-run lock. Direct run/cleanup and foreground publication exclude each other under that lock. +New publications use a closed version3 owner record with independently captured +native host boot time. Live authentication rechecks that boot together with the +existing exact process and filesystem identities. The closed version2 live-owner +decoder remains available without inferring host boot from PID birth or guest +receipt boot. Version2 records reject the new field, and version3 records require +it. Native runtime receipts and the authenticated control/ready wire remain v2; +Compose receipt and owner formats remain unchanged. This provenance alone grants +no dead-owner recovery authority. Cancellation fences startup before admission and subsequent effects. Cleanup retains stop failures by admitted workload name, and retires the publication only after exact cleanup succeeds. Replies bind their requested action and immutable diff --git a/packages/runtime-core/src/provider/graph/native/foreground/owner.rs b/packages/runtime-core/src/provider/graph/native/foreground/owner.rs index 2b401ae14..5444f1850 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/owner.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/owner.rs @@ -33,6 +33,57 @@ struct Record { socket: (u64, u64), lock: (u64, u64), } +/// A distinct closed publication format: old live owners never acquire boot provenance by inference. +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct BootQualifiedRecord { + version: u32, + kind: Kind, + candidate: PathBuf, + review: native_input::Review, + process: ProcessIdentity, + parent: (u64, u64), + socket: (u64, u64), + lock: (u64, u64), + host_boot_micros: u64, +} +impl BootQualifiedRecord { + fn into_record(self) -> (Record, Option) { + ( + Record { + version: self.version, + kind: self.kind, + candidate: self.candidate, + review: self.review, + process: self.process, + parent: self.parent, + socket: self.socket, + lock: self.lock, + }, + Some(self.host_boot_micros), + ) + } +} +fn decode(bytes: &[u8]) -> Result<(Record, Option), CandidateError> { + let value: Value = serde_json::from_slice(bytes).map_err(|_| refused())?; + match value.get("version").and_then(Value::as_u64) { + Some(2) => { + let record: Record = serde_json::from_slice(bytes).map_err(|_| refused())?; + Ok((record, None)) + } + Some(3) => { + let record: BootQualifiedRecord = + serde_json::from_slice(bytes).map_err(|_| refused())?; + if record.host_boot_micros == 0 + || record.process.start_micros < record.host_boot_micros + { + return Err(refused()); + } + Ok(record.into_record()) + } + _ => Err(refused()), + } +} fn id(metadata: &fs::Metadata) -> (u64, u64) { (metadata.dev(), metadata.ino()) } @@ -110,22 +161,30 @@ impl<'a> DirectGuard<'a> { pub(super) struct Pin { root: PathBuf, record: Record, + host_boot_micros: Option, bytes: Vec, file: (u64, u64), } impl Pin { + fn verify_host_boot(&self) -> Result<(), CandidateError> { + if self.host_boot_micros.is_some_and(|boot| { + crate::provider::host_filesystem::host_boot_micros().ok() != Some(boot) + }) { + return Err(refused()); + } + Ok(()) + } pub(super) fn load(candidate: &Candidate, run: &str) -> Result { let root = root(candidate, run)?; state::check_private_directory(&root).map_err(|_| refused())?; let file = fs::symlink_metadata(root.join("owner.json")).map_err(|_| refused())?; let bytes = native_input::read_file(&root.join("owner.json"), 8192)?; - let record: Record = serde_json::from_slice(&bytes).map_err(|_| refused())?; + let (record, host_boot_micros) = decode(&bytes)?; record .review .validate(record.review.scope()) .map_err(|_| refused())?; - if record.version != 2 - || record.candidate != candidate.checkout + if record.candidate != candidate.checkout || record.review.scope().run != run { return Err(refused()); @@ -133,6 +192,7 @@ impl Pin { let pin = Self { root, record, + host_boot_micros, bytes, file: id(&file), }; @@ -145,6 +205,7 @@ impl Pin { /// The caller already authenticated its retained connection before the owner /// retired. Absence grants no new connection, process or cleanup authority. pub(super) fn verify_retired(&self) -> Result<(), CandidateError> { + self.verify_host_boot()?; state::check_private_directory(&self.root).map_err(|_| refused())?; let parent = fs::symlink_metadata(&self.root).map_err(|_| refused())?; let lock = fs::symlink_metadata(self.root.join("operation.lock")).map_err(|_| refused())?; @@ -162,6 +223,7 @@ impl Pin { Ok(()) } pub(super) fn verify(&self) -> Result<(), CandidateError> { + self.verify_host_boot()?; state::check_private_directory(&self.root).map_err(|_| refused())?; let parent = fs::symlink_metadata(&self.root).map_err(|_| refused())?; let socket = fs::symlink_metadata(self.root.join("control.sock")).map_err(|_| refused())?; @@ -217,6 +279,8 @@ impl Publication { review: &native_input::Review, ) -> Result { review.validate(review.scope()).map_err(|_| refused())?; + let host_boot_micros = + crate::provider::host_filesystem::host_boot_micros().map_err(|_| refused())?; let gate = super::super::super::publication_gate::Guard::acquire(candidate)?; let run = review.scope().run; if exists(&super::super::journal::directory(candidate, run)?)? { @@ -232,8 +296,8 @@ impl Publication { fs::set_permissions(root.join("control.sock"), fs::Permissions::from_mode(0o600)) .map_err(|_| refused())?; listener.set_nonblocking(true).map_err(|_| refused())?; - let record = Record { - version: 2, + let record = BootQualifiedRecord { + version: 3, kind: Kind::NativeGraphOwner, candidate: candidate.checkout.clone(), review: review.clone(), @@ -241,11 +305,13 @@ impl Publication { parent: id(&fs::symlink_metadata(&root).map_err(|_| refused())?), socket: id(&fs::symlink_metadata(root.join("control.sock")).map_err(|_| refused())?), lock: lock.identity().map_err(|_| refused())?, + host_boot_micros, }; let bytes = serde_json::to_vec(&record).map_err(|_| refused())?; if bytes.len() > 8192 { return Err(refused()); } + let (record, host_boot_micros) = decode(&bytes)?; let mut file = OpenOptions::new() .write(true) .create_new(true) @@ -259,6 +325,7 @@ impl Publication { let pin = Pin { root, record, + host_boot_micros, bytes, file: id(&file.metadata().map_err(|_| refused())?), }; diff --git a/packages/runtime-core/src/provider/graph/native/foreground/tests.rs b/packages/runtime-core/src/provider/graph/native/foreground/tests.rs index 354bfaadb..d22fe9723 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/tests.rs @@ -291,6 +291,9 @@ fn owner_codec_refuses_legacy_kind_unknown_fields_and_wrong_process_incarnation( ("/candidate", json!("/foreign")), ("/process/start_micros", json!(0)), ("/process/pid", json!(2_000_000)), + ("/host_boot_micros", json!(0)), + ("/host_boot_micros", json!(1)), + ("/host_boot_micros", json!(u64::MAX)), ("/values", json!("private-canary")), ] { let fixture = Fixture::new(); @@ -314,6 +317,46 @@ fn owner_codec_refuses_legacy_kind_unknown_fields_and_wrong_process_incarnation( } } +#[test] +fn publication_three_binds_independent_host_boot_and_keeps_closed_live_version_two() { + let fixture = Fixture::new(); + let review = fixture.receipt().review; + let publication = owner::Publication::bind(&fixture.candidate, &review).unwrap(); + let file = fixture.owner_root().join("owner.json"); + let bytes = fs::read(&file).unwrap(); + let qualified: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(qualified["version"], 3); + let boot = crate::provider::host_filesystem::host_boot_micros().unwrap(); + assert_eq!(qualified["host_boot_micros"], boot); + assert!(qualified["process"]["start_micros"].as_u64().unwrap() >= boot); + owner::Pin::load(&fixture.candidate, RUN).unwrap(); + publication.verify().unwrap(); + assert!(DirectGuard::acquire(&fixture.candidate, RUN).is_err()); + + let mut old = qualified.clone(); + old["version"] = json!(2); + old.as_object_mut().unwrap().remove("host_boot_micros"); + fs::write(&file, serde_json::to_vec(&old).unwrap()).unwrap(); + // Exact legacy live authentication remains available without inventing host boot. + let legacy = owner::Pin::load(&fixture.candidate, RUN).unwrap(); + legacy.connect().unwrap(); + assert!(DirectGuard::acquire(&fixture.candidate, RUN).is_err()); + assert!(publication.verify().is_err()); + + for host_boot in [json!(boot), Value::Null] { + let mut wrong_version = old.clone(); + wrong_version["host_boot_micros"] = host_boot; + fs::write(&file, serde_json::to_vec(&wrong_version).unwrap()).unwrap(); + assert!(owner::Pin::load(&fixture.candidate, RUN).is_err()); + } + let mut missing_boot = old; + missing_boot["version"] = json!(3); + fs::write(&file, serde_json::to_vec(&missing_boot).unwrap()).unwrap(); + assert!(owner::Pin::load(&fixture.candidate, RUN).is_err()); + assert!(file.exists()); + assert!(!fixture.candidate.state_root.join("run/smolvm").exists()); +} + #[test] fn replaced_symlinked_hardlinked_and_public_owner_files_are_not_adopted() { for kind in ["replaced", "symlink", "hardlink", "public"] { From 880e2538e4aabbb21cb8d981ab41ad1a7cb1ca09 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 12:43:48 -0400 Subject: [PATCH 02/47] style(runtime): format native publication decoder --- .../src/provider/graph/native/foreground/owner.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/runtime-core/src/provider/graph/native/foreground/owner.rs b/packages/runtime-core/src/provider/graph/native/foreground/owner.rs index 5444f1850..f8a728ec6 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/owner.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/owner.rs @@ -74,8 +74,7 @@ fn decode(bytes: &[u8]) -> Result<(Record, Option), CandidateError> { Some(3) => { let record: BootQualifiedRecord = serde_json::from_slice(bytes).map_err(|_| refused())?; - if record.host_boot_micros == 0 - || record.process.start_micros < record.host_boot_micros + if record.host_boot_micros == 0 || record.process.start_micros < record.host_boot_micros { return Err(refused()); } @@ -184,9 +183,7 @@ impl Pin { .review .validate(record.review.scope()) .map_err(|_| refused())?; - if record.candidate != candidate.checkout - || record.review.scope().run != run - { + if record.candidate != candidate.checkout || record.review.scope().run != run { return Err(refused()); } let pin = Self { From 8633485c5c883b1c7cf7f420ad58c5ad11be169e Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 12:52:59 -0400 Subject: [PATCH 03/47] fix(runtime): preserve native cleanup progress on retry --- packages/runtime-core/README.md | 5 +- .../src/provider/graph/native/runtime.rs | 46 ++++-- .../provider/graph/native/runtime/tests.rs | 136 ++++++++++++++++++ 3 files changed, 173 insertions(+), 14 deletions(-) diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 9feba8153..95e75b25d 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -99,7 +99,10 @@ changed network policy refuse before cleanup effects. Cleanup preflights every stop, retains terminal observations before deletes, removes the verified empty project network after containers, and retires private payloads only after the bound container is absent. Failed/uncertain attempts retain their -reservations. Shared admission counts native and Compose attempts together; a build +reservations. Cleanup retries preserve committed stop/removal progress: terminal +instances are not stopped again, and fully Removed inventories are inspected without +rewriting their receipt. A restarted stopped instance or reappearing removed resource +refuses before any cleanup effect. Shared admission counts native and Compose attempts together; a build with native support accepts validated, fully removed same-owner history from an older boot for capacity purposes only. Active, malformed or foreign old-boot records still refuse; they grant no inspection, cleanup or restore authority. A build diff --git a/packages/runtime-core/src/provider/graph/native/runtime.rs b/packages/runtime-core/src/provider/graph/native/runtime.rs index cef21d386..71d920098 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime.rs @@ -665,6 +665,14 @@ fn cleanup_using_guarded( guard: Option<&dyn Fn() -> Result<(), CandidateError>>, ) -> Result<(), CandidateError> { check_startup(guard)?; + // A committed cleanup phase is retry authority for this same inventory, never + // permission to move the receipt back to startup or stop an already retired run. + let stopped = matches!( + receipt.phase, + Phase::Stopped | Phase::RemovalIntent | Phase::Removed + ); + let removing = matches!(receipt.phase, Phase::RemovalIntent | Phase::Removed); + let removed = receipt.phase == Phase::Removed; let mut prepared = BTreeMap::new(); let network = receipt .resources @@ -673,7 +681,7 @@ fn cleanup_using_guarded( .clone(); let network_present = inspected(backend, receipt, &network)?.is_some(); if network_present { - if network.id.is_none() { + if network.id.is_none() || removed || network.phase == "removed" { return Err(refused()); } project_network(backend, receipt, None)?; @@ -685,22 +693,28 @@ fn cleanup_using_guarded( .filter(|(_, r)| r.kind == Kind::Container) { if let Some(value) = inspected(backend, receipt, resource)? { - if resource.id.is_none() { + if resource.id.is_none() || removed || resource.phase == "removed" { return Err(refused()); } if network_present { verify_attachment(receipt, &resource.key, &value)?; project_network(backend, receipt, Some(&value))?; } - prepared.insert( - key.clone(), - super::super::shutdown::prepare(resource, &value)?, - ); + let selected = super::super::shutdown::prepare(resource, &value)?; + if stopped && selected.running { + return Err(refused()); + } + prepared.insert(key.clone(), selected); } } check_startup(guard)?; - receipt.phase = Phase::StopIntent; - journal::save(root, receipt)?; + if removed { + return Ok(()); + } + if !stopped { + receipt.phase = Phase::StopIntent; + journal::save(root, receipt)?; + } let stops = prepared .values() .filter(|p| p.running) @@ -712,7 +726,9 @@ fn cleanup_using_guarded( .filter(|(_, prepared)| prepared.running) .map(|(key, prepared)| (prepared.id.as_str(), receipt.resources[key].key.as_str())) .collect::>(); - backend.stop(&stops, &admitted)?; + if !stopped { + backend.stop(&stops, &admitted)?; + } for (key, prepared) in &prepared { let resource = &receipt.resources[key]; let value = inspected(backend, receipt, resource)?.ok_or_else(refused)?; @@ -722,10 +738,14 @@ fn cleanup_using_guarded( } super::super::shutdown::record_terminal(&mut receipt.terminal, key.clone(), terminal); } - receipt.phase = Phase::Stopped; - journal::save(root, receipt)?; - receipt.phase = Phase::RemovalIntent; - journal::save(root, receipt)?; + if !stopped { + receipt.phase = Phase::Stopped; + journal::save(root, receipt)?; + } + if !removing { + receipt.phase = Phase::RemovalIntent; + journal::save(root, receipt)?; + } for key in prepared.keys() { check_startup(guard)?; let resource = &receipt.resources[key]; diff --git a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs index cad693c9a..bfc0012bf 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs @@ -175,6 +175,8 @@ struct FakeState { containers: BTreeMap, networks: BTreeMap, effects: Vec, + stop_batches: usize, + observed_phases: Vec, fail_create: bool, fail_start: bool, fail_delete: bool, @@ -205,6 +207,7 @@ impl Backend for Fake { body: Option<&Value>, ) -> Result { let mut state = self.state.borrow_mut(); + state.observed_phases.push(self.receipt().phase); if method == Method::GET && path.starts_with("/v1.53/networks/") { let selected = path.strip_prefix("/v1.53/networks/").unwrap(); return state @@ -391,6 +394,7 @@ impl Backend for Fake { ) -> Result<(), CandidateError> { assert_eq!(self.receipt().phase, Phase::StopIntent); let mut state = self.state.borrow_mut(); + state.stop_batches += 1; if state.fail_stop { return Err(super::super::super::shutdown::stop_error( crate::provider::engine::StopBatchFailure { @@ -780,6 +784,138 @@ fn cleanup_retry_keeps_observed_stop_request_for_the_same_immutable_id() { ); } +#[test] +fn committed_removal_retry_does_not_repeat_stop_or_regress_the_receipt_phase() { + let fixture = Fixture::new(basic()); + let (graph, mut session) = + fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); + execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + session.backend.state.borrow_mut().fail_delete = true; + assert!(cleanup_using(&session.backend, &mut session.receipt, &session.root).is_err()); + let (mut retained, _) = journal::load(&fixture.candidate, RUN, OWNER, BOOT).unwrap(); + assert_eq!(retained.phase, Phase::RemovalIntent); + let before_stops = session.backend.state.borrow().stop_batches; + session.backend.state.borrow_mut().observed_phases.clear(); + cleanup_using(&session.backend, &mut retained, &session.root).unwrap(); + assert_eq!(retained.phase, Phase::Removed); + let state = session.backend.state.borrow(); + assert_eq!(state.stop_batches, before_stops); + assert!( + state + .observed_phases + .iter() + .all(|phase| *phase == Phase::RemovalIntent) + ); + assert!(retained.terminal["container:web"].stop_requested); +} + +#[test] +fn stopped_cleanup_resumes_only_terminal_instances_without_another_stop_batch() { + let fixture = Fixture::new(basic()); + let (graph, mut session) = + fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); + execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + session.backend.state.borrow_mut().fail_delete = true; + assert!(cleanup_using(&session.backend, &mut session.receipt, &session.root).is_err()); + // Select the durable Stopped boundary with actual matching terminal observations. + let (mut retained, _) = journal::load(&fixture.candidate, RUN, OWNER, BOOT).unwrap(); + retained.phase = Phase::Stopped; + journal::save(&session.root, &retained).unwrap(); + let before_stops = session.backend.state.borrow().stop_batches; + session.backend.state.borrow_mut().observed_phases.clear(); + cleanup_using(&session.backend, &mut retained, &session.root).unwrap(); + assert_eq!(retained.phase, Phase::Removed); + let state = session.backend.state.borrow(); + assert_eq!(state.stop_batches, before_stops); + assert!( + state + .observed_phases + .iter() + .all(|phase| matches!(phase, Phase::Stopped | Phase::RemovalIntent)) + ); +} + +#[test] +fn cleanup_of_removed_inventory_is_read_only_and_keeps_the_committed_bytes() { + let fixture = Fixture::new(basic()); + let (graph, mut session) = + fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); + execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + cleanup_using(&session.backend, &mut session.receipt, &session.root).unwrap(); + let before = fs::read(session.root.join("state.json")).unwrap(); + let effects = session.backend.state.borrow().effects.clone(); + let stops = session.backend.state.borrow().stop_batches; + session.backend.state.borrow_mut().observed_phases.clear(); + cleanup_using(&session.backend, &mut session.receipt, &session.root).unwrap(); + assert_eq!(fs::read(session.root.join("state.json")).unwrap(), before); + let state = session.backend.state.borrow(); + assert_eq!(state.effects, effects); + assert_eq!(state.stop_batches, stops); + assert!( + state + .observed_phases + .iter() + .all(|phase| *phase == Phase::Removed) + ); +} + +#[test] +fn stopped_or_removing_inventory_refuses_a_restarted_instance_before_effects() { + for phase in [Phase::Stopped, Phase::RemovalIntent] { + let fixture = Fixture::new(basic()); + let (graph, mut session) = + fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); + execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + session.backend.state.borrow_mut().fail_delete = true; + assert!(cleanup_using(&session.backend, &mut session.receipt, &session.root).is_err()); + session.receipt.phase = phase; + journal::save(&session.root, &session.receipt).unwrap(); + let name = &session.receipt.resources["container:web"].name; + { + let mut state = session.backend.state.borrow_mut(); + let value = state.containers.get_mut(name).unwrap(); + value["State"]["Running"] = json!(true); + value["State"]["Status"] = json!("running"); + value["State"]["Pid"] = json!(1234); + } + let before = fs::read(session.root.join("state.json")).unwrap(); + let effects = session.backend.state.borrow().effects.clone(); + let stops = session.backend.state.borrow().stop_batches; + assert!(cleanup_using(&session.backend, &mut session.receipt, &session.root).is_err()); + assert_eq!(fs::read(session.root.join("state.json")).unwrap(), before); + assert_eq!(session.backend.state.borrow().effects, effects); + assert_eq!(session.backend.state.borrow().stop_batches, stops); + } +} + +#[test] +fn removed_inventory_refuses_even_matching_resource_reappearance_without_effects() { + for container in [false, true] { + let fixture = Fixture::new(basic()); + let (graph, mut session) = + fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); + execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + let containers = session.backend.state.borrow().containers.clone(); + let networks = session.backend.state.borrow().networks.clone(); + cleanup_using(&session.backend, &mut session.receipt, &session.root).unwrap(); + { + let mut state = session.backend.state.borrow_mut(); + if container { + state.containers = containers; + } else { + state.networks = networks; + } + } + let before = fs::read(session.root.join("state.json")).unwrap(); + let effects = session.backend.state.borrow().effects.clone(); + let stops = session.backend.state.borrow().stop_batches; + assert!(cleanup_using(&session.backend, &mut session.receipt, &session.root).is_err()); + assert_eq!(fs::read(session.root.join("state.json")).unwrap(), before); + assert_eq!(session.backend.state.borrow().effects, effects); + assert_eq!(session.backend.state.borrow().stop_batches, stops); + } +} + #[test] fn cleanup_owner_guard_refuses_before_provider_connection_or_receipt_mutation() { let fixture = Fixture::new(basic()); From e1deb22049904082b93ebd8be6f046ba6f142487 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 12:56:41 -0400 Subject: [PATCH 04/47] fix(runtime): persist fresh retry terminal evidence before delete --- packages/runtime-core/src/provider/graph/native/runtime.rs | 4 +++- .../runtime-core/src/provider/graph/native/runtime/tests.rs | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/runtime-core/src/provider/graph/native/runtime.rs b/packages/runtime-core/src/provider/graph/native/runtime.rs index 71d920098..dfd17dbca 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime.rs @@ -740,8 +740,10 @@ fn cleanup_using_guarded( } if !stopped { receipt.phase = Phase::Stopped; - journal::save(root, receipt)?; } + // A resumed removal still durably records each fresh terminal observation + // before deletion, without moving its already committed phase backward. + journal::save(root, receipt)?; if !removing { receipt.phase = Phase::RemovalIntent; journal::save(root, receipt)?; diff --git a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs index bfc0012bf..8b5dd27e1 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs @@ -794,6 +794,10 @@ fn committed_removal_retry_does_not_repeat_stop_or_regress_the_receipt_phase() { assert!(cleanup_using(&session.backend, &mut session.receipt, &session.root).is_err()); let (mut retained, _) = journal::load(&fixture.candidate, RUN, OWNER, BOOT).unwrap(); assert_eq!(retained.phase, Phase::RemovalIntent); + // Valid bounded state with missing terminal detail must persist fresh evidence + // before Fake's DELETE independently reads the durable journal. + retained.terminal.clear(); + journal::save(&session.root, &retained).unwrap(); let before_stops = session.backend.state.borrow().stop_batches; session.backend.state.borrow_mut().observed_phases.clear(); cleanup_using(&session.backend, &mut retained, &session.root).unwrap(); @@ -806,7 +810,7 @@ fn committed_removal_retry_does_not_repeat_stop_or_regress_the_receipt_phase() { .iter() .all(|phase| *phase == Phase::RemovalIntent) ); - assert!(retained.terminal["container:web"].stop_requested); + assert!(!retained.terminal["container:web"].stop_requested); } #[test] From eeb6b93957b00e888772e5c93d8db380b190314c Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 13:35:39 -0400 Subject: [PATCH 05/47] feat(runtime): select exact dead native foreground recovery evidence --- .../provider/graph/foreground/transport.rs | 2 +- .../src/provider/graph/native/foreground.rs | 1 + .../provider/graph/native/foreground/owner.rs | 198 +++++++- .../graph/native/foreground/recovery.rs | 265 +++++++++++ .../graph/native/foreground/recovery/tests.rs | 448 ++++++++++++++++++ .../src/provider/graph/native/journal.rs | 41 ++ .../src/provider/graph/publication_gate.rs | 15 + 7 files changed, 956 insertions(+), 14 deletions(-) create mode 100644 packages/runtime-core/src/provider/graph/native/foreground/recovery.rs create mode 100644 packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs diff --git a/packages/runtime-core/src/provider/graph/foreground/transport.rs b/packages/runtime-core/src/provider/graph/foreground/transport.rs index 6fe2ac4aa..47f7421d8 100644 --- a/packages/runtime-core/src/provider/graph/foreground/transport.rs +++ b/packages/runtime-core/src/provider/graph/foreground/transport.rs @@ -191,7 +191,7 @@ fn metadata(path: &Path) -> Result, CandidateError> { /// A dead PID alone cannot prove that another process did not inherit the /// listener. A single nonblocking connect must report a refused socket. -fn no_listener(path: &Path) -> Result<(), CandidateError> { +pub(in crate::provider::graph) fn no_listener(path: &Path) -> Result<(), CandidateError> { let bytes = path.as_os_str().as_encoded_bytes(); // SAFETY: sockaddr_un is plain C storage, populated before connect. let mut address: libc::sockaddr_un = unsafe { std::mem::zeroed() }; diff --git a/packages/runtime-core/src/provider/graph/native/foreground.rs b/packages/runtime-core/src/provider/graph/native/foreground.rs index 1b28f3b3d..55161dd75 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground.rs @@ -4,6 +4,7 @@ use crate::provider::graph::foreground::{signals, transport}; use serde::{Deserialize, Serialize}; use std::{cell::Cell, io::Write}; mod owner; +pub mod recovery; pub(super) use owner::DirectGuard; #[cfg(test)] mod tests; diff --git a/packages/runtime-core/src/provider/graph/native/foreground/owner.rs b/packages/runtime-core/src/provider/graph/native/foreground/owner.rs index f8a728ec6..c75c9c394 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/owner.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/owner.rs @@ -174,6 +174,11 @@ impl Pin { Ok(()) } pub(super) fn load(candidate: &Candidate, run: &str) -> Result { + let pin = Self::read(candidate, run)?; + pin.verify()?; + Ok(pin) + } + fn read(candidate: &Candidate, run: &str) -> Result { let root = root(candidate, run)?; state::check_private_directory(&root).map_err(|_| refused())?; let file = fs::symlink_metadata(root.join("owner.json")).map_err(|_| refused())?; @@ -193,7 +198,6 @@ impl Pin { bytes, file: id(&file), }; - pin.verify()?; Ok(pin) } pub(super) fn review(&self) -> &native_input::Review { @@ -220,6 +224,21 @@ impl Pin { Ok(()) } pub(super) fn verify(&self) -> Result<(), CandidateError> { + self.verify_files()?; + let expected = &self.record.process; + // SAFETY: geteuid has no preconditions; same-user peer authority remains native. + if expected.uid != unsafe { libc::geteuid() } { + return Err(refused()); + } + identity::verify( + expected, + &identity::observe(expected.pid).map_err(|_| refused())?, + &expected.executable, + expected.uid, + ) + .map_err(|_| refused()) + } + fn verify_files(&self) -> Result<(), CandidateError> { self.verify_host_boot()?; state::check_private_directory(&self.root).map_err(|_| refused())?; let parent = fs::symlink_metadata(&self.root).map_err(|_| refused())?; @@ -240,18 +259,7 @@ impl Pin { { return Err(refused()); } - let expected = &self.record.process; - // SAFETY: geteuid has no preconditions; same-user peer authority remains native. - if expected.uid != unsafe { libc::geteuid() } { - return Err(refused()); - } - identity::verify( - expected, - &identity::observe(expected.pid).map_err(|_| refused())?, - &expected.executable, - expected.uid, - ) - .map_err(|_| refused()) + Ok(()) } pub(super) fn connect(&self) -> Result { self.verify()?; @@ -265,6 +273,170 @@ impl Pin { } } +/// Exact value-free publication bytes and inode, retained by a durable recovery intent. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct RecoverySelection { + bytes: String, + file: (u64, u64), +} +impl RecoverySelection { + pub(super) fn fingerprint(&self) -> String { + format!("{:x}", Sha256::digest(self.bytes.as_bytes())) + } + fn record(&self, candidate: &Candidate, run: &str) -> Result<(Record, u64), CandidateError> { + if self.bytes.is_empty() || self.bytes.len() > 8192 { + return Err(refused()); + } + let (record, boot) = decode(self.bytes.as_bytes())?; + let boot = boot.ok_or_else(refused)?; + record + .review + .validate(record.review.scope()) + .map_err(|_| refused())?; + // SAFETY: geteuid has no preconditions; private state remains same-user. + if record.version != 3 + || record.candidate != candidate.checkout + || record.review.scope().run != run + || record.process.uid != unsafe { libc::geteuid() } + || crate::provider::host_filesystem::host_boot_micros().ok() != Some(boot) + { + return Err(refused()); + } + Ok((record, boot)) + } +} +/// Recovery keeps gate → original run lock for its entire lease. Ordinary live +/// Pin and DirectGuard remain strict and grant no dead-publication authority. +pub(super) struct RecoveryLease<'a> { + candidate: &'a Candidate, + run: &'a str, + root: PathBuf, + gate: super::super::super::publication_gate::Guard, + lock: state::Lock, + selection: RecoverySelection, +} +impl<'a> RecoveryLease<'a> { + pub(super) fn acquire( + candidate: &'a Candidate, + run: &'a str, + saved: Option, + socket_retirement_admitted: bool, + owner_retirement_admitted: bool, + ) -> Result { + let gate = super::super::super::publication_gate::Guard::acquire_existing(candidate)?; + let root = root(candidate, run)?; + let lock = state::Lock::acquire_existing(&root)?; + let selection = match saved { + Some(value) => value, + None => { + let pin = Pin::read(candidate, run)?; + if pin.host_boot_micros.is_none() { + return Err(refused()); + } + RecoverySelection { + bytes: String::from_utf8(pin.bytes).map_err(|_| refused())?, + file: pin.file, + } + } + }; + let lease = Self { + candidate, + run, + root, + gate, + lock, + selection, + }; + lease.verify(socket_retirement_admitted, owner_retirement_admitted)?; + Ok(lease) + } + pub(super) fn selected(&self) -> &RecoverySelection { + &self.selection + } + pub(super) fn review(&self) -> Result { + self.selection + .record(self.candidate, self.run) + .map(|(record, _)| record.review) + } + pub(super) fn host_boot_micros(&self) -> Result { + self.selection + .record(self.candidate, self.run) + .map(|(_, boot)| boot) + } + fn archive_path(&self, socket: bool) -> PathBuf { + let fingerprint = self.selection.fingerprint(); + self.root.join(if socket { + format!("control-{}.retired.sock", &fingerprint[..24]) + } else { + format!("owner-{}.retired.json", &fingerprint[..24]) + }) + } + fn selected_path( + &self, + socket: bool, + admitted: bool, + expected: (u64, u64), + ) -> Result<(PathBuf, bool), CandidateError> { + let original = self + .root + .join(if socket { "control.sock" } else { "owner.json" }); + let archived = self.archive_path(socket); + let (path, original) = match (exists(&original)?, exists(&archived)?) { + (true, false) => (original, true), + (false, true) if admitted => (archived, false), + _ => return Err(refused()), + }; + let metadata = fs::symlink_metadata(&path).map_err(|_| refused())?; + if !private(&metadata) + || id(&metadata) != expected + || (socket && !metadata.file_type().is_socket()) + || (!socket && (!metadata.is_file() || metadata.nlink() != 1)) + { + return Err(refused()); + } + Ok((path, original)) + } + /// Return which original paths remain. Retired paths require a pre-existing + /// durable phase; missing both original and archive always refuses. + pub(super) fn verify( + &self, + socket_admitted: bool, + owner_admitted: bool, + ) -> Result<(bool, bool), CandidateError> { + self.gate.verify(self.candidate)?; + let (record, _) = self.selection.record(self.candidate, self.run)?; + state::check_private_directory(&self.root).map_err(|_| refused())?; + if id(&fs::symlink_metadata(&self.root).map_err(|_| refused())?) != record.parent + || self.lock.identity()? != record.lock + || identity::alive(record.process.pid).map_err(|_| refused())? + { + return Err(refused()); + } + super::super::super::host_pin_recovery::exact_lock_path(&self.root, &self.lock)?; + let (socket, socket_original) = self.selected_path(true, socket_admitted, record.socket)?; + let (owner, owner_original) = + self.selected_path(false, owner_admitted, self.selection.file)?; + if socket_original && !owner_original + || native_input::read_file(&owner, 8192)? != self.selection.bytes.as_bytes() + { + return Err(refused()); + } + transport::no_listener(&socket)?; + if self.selected_path(true, socket_admitted, record.socket)? != (socket, socket_original) + || self.selected_path(false, owner_admitted, self.selection.file)? + != (owner.clone(), owner_original) + || native_input::read_file(&owner, 8192)? != self.selection.bytes.as_bytes() + || identity::alive(record.process.pid).map_err(|_| refused())? + { + return Err(refused()); + } + self.selection.record(self.candidate, self.run)?; + self.gate.verify(self.candidate)?; + Ok((socket_original, owner_original)) + } +} + pub(super) struct Publication { listener: UnixListener, pin: Pin, diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs new file mode 100644 index 000000000..1ac346fac --- /dev/null +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs @@ -0,0 +1,265 @@ +//! Explicit native recovery selectors and durable monotonic intent. Live control +//! and ordinary DirectGuard never acquire authority from a dead publication. +use super::*; +use sha2::{Digest, Sha256}; +use std::{ + fs, + os::unix::fs::MetadataExt, + path::{Path, PathBuf}, +}; + +const FILE: &str = "live-owner-recovery.json"; +const LIMIT: usize = 128 * 1024; +fn refused() -> CandidateError { + error( + "native_graph_live_owner_recovery", + "Native same-boot recovery requires the exact complete Ready selection, dead boot-qualified publisher and unchanged durable progress; evidence retained.", + ) +} +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} +fn id(path: &Path) -> Result<(u64, u64), CandidateError> { + let metadata = fs::symlink_metadata(path).map_err(|_| refused())?; + Ok((metadata.dev(), metadata.ino())) +} +fn exists(path: &Path) -> Result { + match fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(_) => Err(refused()), + } +} +fn directory(candidate: &Candidate, run: &str) -> Result { + let root = journal::directory(candidate, run)?; + for path in root + .ancestors() + .take_while(|path| path.starts_with(&candidate.state_root)) + { + state::check_private_directory(path).map_err(|_| refused())?; + } + if exists(&root.join("state.pending"))? || exists(&root.join("live-owner-recovery.pending"))? { + return Err(refused()); + } + Ok(root) +} +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +enum Progress { + Cleanup, + EnvironmentRetired, + SocketRetirementIntent, + SocketRetired, + OwnerRetirementIntent, + Complete, +} +impl Progress { + fn verify(&self, lease: &owner::RecoveryLease<'_>) -> Result<(), CandidateError> { + let (socket_original, owner_original) = lease + .verify( + *self >= Self::SocketRetirementIntent, + *self >= Self::OwnerRetirementIntent, + ) + .map_err(|_| refused())?; + if (*self < Self::SocketRetirementIntent && (!socket_original || !owner_original)) + || (*self == Self::SocketRetirementIntent && !owner_original) + || (*self == Self::SocketRetired && (socket_original || !owner_original)) + || (*self == Self::OwnerRetirementIntent && socket_original) + || (*self == Self::Complete && (socket_original || owner_original)) + { + return Err(refused()); + } + Ok(()) + } +} +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +enum IntentKind { + NativeGraphLiveOwnerRecovery, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Intent { + version: u8, + kind: IntentKind, + run: String, + journal_parent: (u64, u64), + original: String, + receipt_sha256: String, + publication: owner::RecoverySelection, + progress: Progress, + receipt_progress: u8, + resource_progress: BTreeMap, +} +fn receipt_progress(receipt: &Receipt) -> Result { + match receipt.phase { + Phase::ReadyObserved => Ok(0), + Phase::StopIntent => Ok(1), + Phase::Stopped => Ok(2), + Phase::RemovalIntent => Ok(3), + Phase::Removed => Ok(4), + _ => Err(refused()), + } +} +fn resource_progress(receipt: &Receipt) -> Result, CandidateError> { + receipt + .resources + .iter() + .map(|(key, resource)| { + let rank = match (&resource.kind, resource.phase.as_str()) { + (Kind::Container, "started") | (Kind::Network, "created") => 0, + (Kind::Network, "remove-intent") => 1, + (Kind::Container | Kind::Network, "removed") => 2, + _ => return Err(refused()), + }; + Ok((key.clone(), rank)) + }) + .collect() +} +impl Intent { + fn original(&self, run: &str) -> Result { + if self.version != 1 + || self.run != run + || self.original.len() > 64 * 1024 + || !super::super::super::hex(&self.receipt_sha256, 64) + || digest(self.original.as_bytes()) != self.receipt_sha256 + || self.receipt_progress > 4 + { + return Err(refused()); + } + let receipt: Receipt = serde_json::from_str(&self.original).map_err(|_| refused())?; + receipt.require_recovery_ready().map_err(|_| refused())?; + if receipt.review.scope().run != run + || self.resource_progress.keys().ne(receipt.resources.keys()) + || self.resource_progress.iter().any(|(key, rank)| { + *rank > 2 || (*rank == 1 && receipt.resources[key].kind != Kind::Network) + }) + { + return Err(refused()); + } + Ok(receipt) + } + fn current( + &self, + root: &Path, + original: &Receipt, + current: &Receipt, + bytes: &str, + ) -> Result<(), CandidateError> { + current.check_binding(original).map_err(|_| refused())?; + let rank = receipt_progress(current)?; + let resources = resource_progress(current)?; + if id(root)? != self.journal_parent + || rank < self.receipt_progress + || (rank == 0 && digest(bytes.as_bytes()) != self.receipt_sha256) + || (self.progress != Progress::Cleanup && rank != 4) + || resources + .iter() + .any(|(key, rank)| *rank < self.resource_progress[key]) + { + return Err(refused()); + } + Ok(()) + } +} +#[derive(Serialize)] +#[serde(rename_all = "kebab-case")] +enum SelectionKind { + NativeGraphRecoverySelection, +} +/// Read-only original selectors. They prove no current guest boot or resource +/// condition and are independently re-admitted by effectful cleanup. +#[derive(Serialize)] +pub struct Selection { + version: u8, + kind: SelectionKind, + run: String, + receipt: Receipt, + receipt_sha256: String, + owner_sha256: String, + host_boot_micros: u64, +} +/// Select only a dead version3 complete Ready publication, or its unchanged +/// committed recovery intent. No file is created or repaired, and no provider +/// operation, private acquisition, signal or cleanup is performed. +pub fn select(candidate: &Candidate, run: &str) -> Result { + let root = directory(candidate, run)?; + let root_identity = id(&root)?; + let intent_path = root.join(FILE); + let witness = if exists(&intent_path)? { + let identity = id(&intent_path)?; + let bytes = native_input::read_file(&intent_path, LIMIT).map_err(|_| refused())?; + if id(&intent_path)? != identity { + return Err(refused()); + } + Some((identity, bytes)) + } else { + None + }; + let intent: Option = witness + .as_ref() + .map(|(_, bytes)| serde_json::from_slice(bytes).map_err(|_| refused())) + .transpose()?; + let lease = owner::RecoveryLease::acquire( + candidate, + run, + intent.as_ref().map(|intent| intent.publication.clone()), + intent + .as_ref() + .is_some_and(|intent| intent.progress >= Progress::SocketRetirementIntent), + intent + .as_ref() + .is_some_and(|intent| intent.progress >= Progress::OwnerRetirementIntent), + )?; + let review = lease.review().map_err(|_| refused())?; + let (current, current_root, bytes) = + journal::read_recovery(candidate, &review).map_err(|_| refused())?; + let (original, receipt_sha256) = match &intent { + Some(intent) => { + let original = intent.original(run)?; + intent.current(¤t_root, &original, ¤t, &bytes)?; + intent.progress.verify(&lease)?; + (original, intent.receipt_sha256.clone()) + } + None => { + current.require_recovery_ready().map_err(|_| refused())?; + Progress::Cleanup.verify(&lease)?; + (current, digest(bytes.as_bytes())) + } + }; + // Recheck immutable intent bytes/inode, publication and journal after all reads. + let (again, _, current_bytes) = + journal::read_recovery(candidate, &review).map_err(|_| refused())?; + if current_bytes != bytes { + return Err(refused()); + } + if directory(candidate, run)? != root || id(&root)? != root_identity { + return Err(refused()); + } + match witness { + Some((identity, original_bytes)) + if id(&intent_path)? == identity + && native_input::read_file(&intent_path, LIMIT).map_err(|_| refused())? + == original_bytes => {} + None if !exists(&intent_path)? => {} + _ => return Err(refused()), + } + if let Some(intent) = &intent { + intent.current(&root, &original, &again, ¤t_bytes)?; + intent.progress.verify(&lease)?; + } else { + Progress::Cleanup.verify(&lease)?; + } + Ok(Selection { + version: 1, + kind: SelectionKind::NativeGraphRecoverySelection, + run: run.into(), + receipt: original, + receipt_sha256, + owner_sha256: lease.selected().fingerprint(), + host_boot_micros: lease.host_boot_micros().map_err(|_| refused())?, + }) +} + +#[cfg(test)] +mod tests; diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs new file mode 100644 index 000000000..3b73952bc --- /dev/null +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs @@ -0,0 +1,448 @@ +use super::*; +use hack_config_compiler::environment::EnvMetadata; +use std::{ + fs::OpenOptions, + io::Write, + os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt}, + process::{Child, Command, Stdio}, + sync::atomic::{AtomicU64, Ordering}, + time::Instant, +}; + +const RUN: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const OWNER: &str = "cccccccccccccccccccccccccccccccc"; +const PUBLISHER: &str = "provider::graph::native::foreground::recovery::tests::publisher_fixture"; +struct Fixture { + root: PathBuf, + candidate: Candidate, + remove: bool, +} +impl Fixture { + fn new() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(0); + let root = std::env::temp_dir().canonicalize().unwrap().join(format!( + "native-selector-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + for path in [ + &root, + &root.join("home"), + &root.join("project"), + &root.join("project/.hack"), + ] { + fs::DirBuilder::new().mode(0o700).create(path).unwrap(); + } + fs::write( + root.join("project/.hack/hack.project.json"), + json!({ + "schema_version":1,"name":"selector","services":{"web":{ + "image":format!("sha256:{}", "d".repeat(64)), + "command":{"exec":["/bin/echo","$EXACT"]} + }} + }) + .to_string(), + ) + .unwrap(); + Self::at(root, true) + } + fn at(root: PathBuf, remove: bool) -> Self { + state::check_private_directory(&root).unwrap(); + Self { + candidate: Candidate::discover(&root.join("home")).unwrap(), + root, + remove, + } + } + fn ready(&self) -> Receipt { + let metadata: EnvMetadata = serde_json::from_value(json!({ + "metadata_version":1,"overlay":null,"overlay_exists":false, + "workloads":{"web":{}},"inactive_scopes":[] + })) + .unwrap(); + let prepared = selection::select( + &self.candidate, + selection::Options { + project: &self.root.join("project"), + branch: Some("fixture"), + run: RUN, + profiles: &[], + explicit_overlay: None, + metadata, + deadline: Instant::now() + Duration::from_secs(60), + }, + ) + .unwrap() + .prepare(&self.candidate, &BTreeMap::new()) + .unwrap(); + let config = configuration(prepared.input(), OWNER).unwrap(); + let mut receipt = + Receipt::preparing(&config, OWNER, "12345678-abcd-abcd-abcd-123456789abc").unwrap(); + receipt.phase = Phase::ReadyObserved; + for (index, resource) in receipt.resources.values_mut().enumerate() { + resource.id = Some(format!("{:064x}", index + 1)); + resource.phase = if resource.kind == Kind::Network { + "created" + } else { + "started" + } + .into(); + } + receipt.require_recovery_ready().unwrap(); + receipt + } + fn publisher(&self, live: bool) -> Child { + Command::new(std::env::current_exe().unwrap()) + .args(["--exact", PUBLISHER, "--nocapture"]) + .env("HACK_NATIVE_SELECTOR_ROOT", &self.root) + .env("HACK_NATIVE_SELECTOR_LIVE", if live { "1" } else { "0" }) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap() + } + fn published(&self) { + let deadline = Instant::now() + Duration::from_secs(5); + while !self.root.join("published").exists() { + assert!( + Instant::now() < deadline, + "fixture publication did not settle" + ); + std::thread::sleep(Duration::from_millis(5)); + } + } + fn dead(&self) { + let mut child = self.publisher(false); + assert!(child.wait().unwrap().success()); + self.published(); + } + fn owner_root(&self) -> PathBuf { + owner::root(&self.candidate, RUN).unwrap() + } + fn journal_root(&self) -> PathBuf { + journal::directory(&self.candidate, RUN).unwrap() + } + fn receipt(&self) -> Receipt { + serde_json::from_slice(&fs::read(self.journal_root().join("state.json")).unwrap()).unwrap() + } + fn intent(&self) -> Intent { + let lease = + owner::RecoveryLease::acquire(&self.candidate, RUN, None, false, false).unwrap(); + let original = fs::read_to_string(self.journal_root().join("state.json")).unwrap(); + let receipt = self.receipt(); + Intent { + version: 1, + kind: IntentKind::NativeGraphLiveOwnerRecovery, + run: RUN.into(), + journal_parent: id(&self.journal_root()).unwrap(), + receipt_sha256: digest(original.as_bytes()), + original, + publication: lease.selected().clone(), + progress: Progress::Cleanup, + receipt_progress: 0, + resource_progress: resource_progress(&receipt).unwrap(), + } + } + fn store(&self, intent: &Intent) { + state::write(&self.journal_root().join(FILE), intent).unwrap(); + } + fn assert_refused_unchanged(&self) { + let original_owner = fs::read(self.owner_root().join("owner.json")).unwrap(); + let original_state = fs::read(self.journal_root().join("state.json")).unwrap(); + let lock = id(&self.owner_root().join("operation.lock")).unwrap(); + assert!(select(&self.candidate, RUN).is_err()); + assert_eq!( + fs::read(self.owner_root().join("owner.json")).unwrap(), + original_owner + ); + assert_eq!( + fs::read(self.journal_root().join("state.json")).unwrap(), + original_state + ); + assert_eq!(id(&self.owner_root().join("operation.lock")).unwrap(), lock); + assert!(!self.candidate.state_root.join("run/smolvm").exists()); + } +} +impl Drop for Fixture { + fn drop(&mut self) { + if self.remove { + let _ = fs::remove_dir_all(self.owner_root()); + fs::remove_dir_all(&self.root).unwrap(); + } + } +} +struct RetainedChild(Child); +impl Drop for RetainedChild { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +/// A real process owns the ordinary publication and lock. No engine is used; +/// the Ready receipt is a typed real-compiler fixture, not a runtime claim. +#[test] +fn publisher_fixture() { + let Some(root) = std::env::var_os("HACK_NATIVE_SELECTOR_ROOT") else { + return; + }; + let fixture = Fixture::at(PathBuf::from(root), false); + let receipt = fixture.ready(); + let _publication = owner::Publication::bind(&fixture.candidate, &receipt.review).unwrap(); + journal::reserve(&fixture.candidate, &receipt).unwrap(); + let mut marker = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(fixture.root.join("published")) + .unwrap(); + marker.write_all(b"published").unwrap(); + marker.sync_all().unwrap(); + if std::env::var_os("HACK_NATIVE_SELECTOR_LIVE").as_deref() == Some(std::ffi::OsStr::new("1")) { + std::thread::sleep(Duration::from_secs(30)); + } + // Deliberately retain the original paths after this exact process exits. +} + +#[test] +fn dead_complete_publication_selects_original_raw_hash_without_creating_authority() { + let fixture = Fixture::new(); + fixture.dead(); + let state_path = fixture.journal_root().join("state.json"); + let original = fs::read(&state_path).unwrap(); + let owner_path = fixture.owner_root().join("owner.json"); + let original_owner = fs::read(&owner_path).unwrap(); + let state_id = id(&state_path).unwrap(); + let owner_id = id(&owner_path).unwrap(); + let lock_id = id(&fixture.owner_root().join("operation.lock")).unwrap(); + let selected = select(&fixture.candidate, RUN).unwrap(); + assert_eq!(selected.receipt_sha256, digest(&original)); + assert_eq!(selected.owner_sha256, digest(&original_owner)); + assert_ne!( + selected.receipt_sha256, + digest(&serde_json::to_vec(&selected.receipt).unwrap()) + ); + assert_eq!( + selected.host_boot_micros, + crate::provider::host_filesystem::host_boot_micros().unwrap() + ); + assert_eq!(fs::read(&state_path).unwrap(), original); + assert_eq!(id(&state_path).unwrap(), state_id); + assert_eq!(fs::read(&owner_path).unwrap(), original_owner); + assert_eq!(id(&owner_path).unwrap(), owner_id); + assert_eq!( + id(&fixture.owner_root().join("operation.lock")).unwrap(), + lock_id + ); + assert!(!fixture.journal_root().join(FILE).exists()); + assert!(!fixture.candidate.state_root.join("run/smolvm").exists()); + assert_eq!( + serde_json::to_value(&selected) + .unwrap() + .as_object() + .unwrap() + .len(), + 7 + ); +} + +#[test] +fn live_exact_publisher_never_yields_dead_recovery_authority() { + let fixture = Fixture::new(); + let mut child = RetainedChild(fixture.publisher(true)); + fixture.published(); + assert!(child.0.try_wait().unwrap().is_none()); + fixture.assert_refused_unchanged(); + assert!(child.0.try_wait().unwrap().is_none()); + child.0.kill().unwrap(); + child.0.wait().unwrap(); + assert!(select(&fixture.candidate, RUN).is_ok()); +} + +#[test] +fn dead_old_two_wrong_boot_and_unknown_publication_fields_refuse() { + for case in ["old-two", "wrong-boot", "unknown"] { + let fixture = Fixture::new(); + fixture.dead(); + let path = fixture.owner_root().join("owner.json"); + let mut owner: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + match case { + "old-two" => { + owner["version"] = json!(2); + owner.as_object_mut().unwrap().remove("host_boot_micros"); + } + "wrong-boot" => { + owner["host_boot_micros"] = json!(owner["host_boot_micros"].as_u64().unwrap() + 1) + } + "unknown" => owner["cleanup_authority"] = json!(true), + _ => unreachable!(), + } + fs::write(&path, owner.to_string()).unwrap(); + fixture.assert_refused_unchanged(); + } +} + +#[test] +fn partial_failed_missing_id_or_pending_state_never_creates_an_intent() { + for case in [ + "preparing", + "failed", + "missing-id", + "state-pending", + "intent-pending", + ] { + let fixture = Fixture::new(); + fixture.dead(); + let root = fixture.journal_root(); + let path = root.join("state.json"); + let mut receipt: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + match case { + "preparing" => receipt["phase"] = json!("preparing"), + "failed" => { + receipt["failure"] = json!({"service":"web","observation":{"state":"dead"}}) + } + "missing-id" => receipt["resources"]["container:web"]["id"] = Value::Null, + "state-pending" => fs::write(root.join("state.pending"), b"retained").unwrap(), + "intent-pending" => { + fs::write(root.join("live-owner-recovery.pending"), b"retained").unwrap() + } + _ => unreachable!(), + } + if !case.ends_with("pending") { + fs::write(&path, receipt.to_string()).unwrap(); + } + fixture.assert_refused_unchanged(); + assert!(!root.join(FILE).exists()); + } +} + +#[test] +fn replaced_owner_socket_and_operation_lock_refuse_without_repair() { + for name in ["owner.json", "control.sock", "operation.lock"] { + let fixture = Fixture::new(); + fixture.dead(); + let path = fixture.owner_root().join(name); + let original = if name == "owner.json" { + Some(fs::read(&path).unwrap()) + } else { + None + }; + fs::rename(&path, path.with_extension("retained")).unwrap(); + if name == "control.sock" { + let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + assert!(select(&fixture.candidate, RUN).is_err()); + drop(listener); + } else { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path) + .unwrap(); + file.write_all(original.as_deref().unwrap_or(b"")).unwrap(); + fixture.assert_refused_unchanged(); + } + } +} + +#[test] +fn committed_intent_keeps_original_hash_and_inventory_through_cleanup_progress() { + let fixture = Fixture::new(); + fixture.dead(); + let mut intent = fixture.intent(); + let original_receipt = intent.receipt_sha256.clone(); + let original_owner = intent.publication.fingerprint(); + fixture.store(&intent); + for phase in [ + Phase::StopIntent, + Phase::Stopped, + Phase::RemovalIntent, + Phase::Removed, + ] { + let mut receipt = fixture.receipt(); + receipt.phase = phase; + if receipt.phase == Phase::Removed { + for resource in receipt.resources.values_mut() { + resource.phase = "removed".into(); + } + } + journal::save(&fixture.journal_root(), &receipt).unwrap(); + let selected = select(&fixture.candidate, RUN).unwrap(); + assert_eq!(selected.receipt.phase, Phase::ReadyObserved); + assert_eq!(selected.receipt_sha256, original_receipt); + assert_eq!(selected.owner_sha256, original_owner); + intent.receipt_progress = receipt_progress(&receipt).unwrap(); + intent.resource_progress = resource_progress(&receipt).unwrap(); + fixture.store(&intent); + } + let mut bad = fixture.receipt(); + bad.phase = Phase::RemovalIntent; + journal::save(&fixture.journal_root(), &bad).unwrap(); + fixture.assert_refused_unchanged(); +} + +#[test] +fn changed_inventory_and_regressed_resource_progress_refuse_under_original_intent() { + for case in ["id", "owner", "boot", "resource-regression"] { + let fixture = Fixture::new(); + fixture.dead(); + let mut intent = fixture.intent(); + let mut current = fixture.receipt(); + current.phase = Phase::RemovalIntent; + intent.receipt_progress = 3; + match case { + "id" => current.resources.get_mut("container:web").unwrap().id = Some("e".repeat(64)), + "owner" => current.owner = "e".repeat(32), + "boot" => current.boot = "12345678-abcd-abcd-abcd-223456789abc".into(), + "resource-regression" => { + intent.resource_progress.insert("container:web".into(), 2); + } + _ => unreachable!(), + } + fixture.store(&intent); + journal::save(&fixture.journal_root(), ¤t).unwrap(); + fixture.assert_refused_unchanged(); + } +} + +#[test] +fn retired_publication_paths_require_prior_exact_phase_and_never_missing_both() { + let fixture = Fixture::new(); + fixture.dead(); + let mut intent = fixture.intent(); + let mut receipt = fixture.receipt(); + receipt.phase = Phase::Removed; + for resource in receipt.resources.values_mut() { + resource.phase = "removed".into(); + } + journal::save(&fixture.journal_root(), &receipt).unwrap(); + intent.receipt_progress = 4; + intent.resource_progress = resource_progress(&receipt).unwrap(); + fixture.store(&intent); + let prefix = &intent.publication.fingerprint()[..24]; + let socket_archive = fixture + .owner_root() + .join(format!("control-{prefix}.retired.sock")); + let owner_archive = fixture + .owner_root() + .join(format!("owner-{prefix}.retired.json")); + fs::rename(fixture.owner_root().join("control.sock"), &socket_archive).unwrap(); + assert!(select(&fixture.candidate, RUN).is_err()); + intent.progress = Progress::SocketRetirementIntent; + fixture.store(&intent); + assert!(select(&fixture.candidate, RUN).is_ok()); + intent.progress = Progress::SocketRetired; + fixture.store(&intent); + fs::rename(fixture.owner_root().join("owner.json"), &owner_archive).unwrap(); + assert!(select(&fixture.candidate, RUN).is_err()); + intent.progress = Progress::OwnerRetirementIntent; + fixture.store(&intent); + assert!(select(&fixture.candidate, RUN).is_ok()); + intent.progress = Progress::Complete; + fixture.store(&intent); + let final_selection = select(&fixture.candidate, RUN).unwrap(); + assert_eq!(final_selection.receipt_sha256, intent.receipt_sha256); + fs::remove_file(socket_archive).unwrap(); + assert!(select(&fixture.candidate, RUN).is_err()); +} diff --git a/packages/runtime-core/src/provider/graph/native/journal.rs b/packages/runtime-core/src/provider/graph/native/journal.rs index ee5baf822..5a9b85572 100644 --- a/packages/runtime-core/src/provider/graph/native/journal.rs +++ b/packages/runtime-core/src/provider/graph/native/journal.rs @@ -104,6 +104,20 @@ impl Receipt { pub fn resources(&self) -> &BTreeMap { &self.resources } + #[cfg(target_os = "macos")] + pub(super) fn require_recovery_ready(&self) -> Result<(), CandidateError> { + self.validate(self.review.scope().run, &self.owner)?; + if self.phase != Phase::ReadyObserved + || self.failure.is_some() + || self + .resources + .values() + .any(|resource| resource.id.is_none()) + { + return Err(refused()); + } + Ok(()) + } pub(super) fn preparing( config: &Configuration, owner: &str, @@ -341,6 +355,33 @@ pub(super) fn read_control( } Ok(receipt) } +/// Retain original bytes for explicit recovery selectors without reconstructing +/// a digest from another wire projection. Pending state remains a refusal. +#[cfg(target_os = "macos")] +pub(super) fn read_recovery( + candidate: &Candidate, + review: &native_input::Review, +) -> Result<(Receipt, PathBuf, String), CandidateError> { + use std::os::unix::fs::MetadataExt; + let receipt = read_control(candidate, review)?; + let root = directory(candidate, review.scope().run)?; + let path = root.join("state.json"); + let before = fs::symlink_metadata(&path).map_err(|_| refused())?; + let bytes = native_input::read_file(&path, LIMIT)?; + let after = fs::symlink_metadata(&path).map_err(|_| refused())?; + let current: Receipt = serde_json::from_slice(&bytes).map_err(|_| refused())?; + if (before.dev(), before.ino()) != (after.dev(), after.ino()) + || serde_json::to_vec(¤t).map_err(|_| refused())? + != serde_json::to_vec(&receipt).map_err(|_| refused())? + { + return Err(refused()); + } + Ok(( + receipt, + root, + String::from_utf8(bytes).map_err(|_| refused())?, + )) +} fn load_validated( candidate: &Candidate, run: &str, diff --git a/packages/runtime-core/src/provider/graph/publication_gate.rs b/packages/runtime-core/src/provider/graph/publication_gate.rs index a43db0dcc..0ca02f9fd 100644 --- a/packages/runtime-core/src/provider/graph/publication_gate.rs +++ b/packages/runtime-core/src/provider/graph/publication_gate.rs @@ -11,6 +11,21 @@ pub(crate) struct Guard { } impl Guard { + /// Selection/recovery observes an existing publication gate without creating authority. + #[cfg(all(target_os = "macos", feature = "native-config-plan"))] + pub(crate) fn acquire_existing(candidate: &Candidate) -> Result { + let root = candidate.state_root.join("run/graph-publication-gate"); + let lock = state::Lock::acquire_existing(&root)?; + let metadata = fs::symlink_metadata(&root).map_err(state::io)?; + let guard = Self { + root, + directory: (metadata.dev(), metadata.ino()), + lock, + }; + guard.verify(candidate)?; + Ok(guard) + } + pub(crate) fn acquire(candidate: &Candidate) -> Result { let root = candidate.state_root.join("run/graph-publication-gate"); let lock = state::Lock::acquire(&root)?; From 77a43dfe4ddbd856a02d4fbc1860e0babe4e5741 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 13:39:35 -0400 Subject: [PATCH 06/47] test(runtime): bind recovery file replacement to saved selection --- packages/runtime-core/README.md | 9 ++++++ .../graph/native/foreground/recovery.rs | 4 ++- .../graph/native/foreground/recovery/tests.rs | 32 ++++++++++++++++++- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 95e75b25d..14c9b9404 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -122,6 +122,15 @@ receipt boot. Version2 records reject the new field, and version3 records requir it. Native runtime receipts and the authenticated control/ready wire remain v2; Compose receipt and owner formats remain unchanged. This provenance alone grants no dead-owner recovery authority. +The inactive read-only recovery selector admits only a complete Ready journal and +a dead version3 publication on the same native host boot. It captures the current +private owner bytes and inode; its raw SHA is a selector for later independent +admission, not an external identity anchor from before that capture. After a +durable intent, the saved inode and digest refuse replacement. Cleanup progress +retains the original Ready selectors and exact inventory. Pending writes refuse; +retired paths require the intent's preceding phase and unchanged archived inode. +This library selector creates no intent, connects no provider and has no CLI or +frontend recovery activation. Cancellation fences startup before admission and subsequent effects. Cleanup retains stop failures by admitted workload name, and retires the publication only after exact cleanup succeeds. Replies bind their requested action and immutable diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs index 1ac346fac..b438274f1 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs @@ -168,7 +168,9 @@ enum SelectionKind { NativeGraphRecoverySelection, } /// Read-only original selectors. They prove no current guest boot or resource -/// condition and are independently re-admitted by effectful cleanup. +/// condition and are independently re-admitted by effectful cleanup. The first +/// selection captures the current private publication, not an external prior +/// owner-file inode; a committed intent anchors that admitted inode and digest. #[derive(Serialize)] pub struct Selection { version: u8, diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs index 3b73952bc..281a40208 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs @@ -317,10 +317,15 @@ fn partial_failed_missing_id_or_pending_state_never_creates_an_intent() { } #[test] -fn replaced_owner_socket_and_operation_lock_refuse_without_repair() { +fn replaced_selected_owner_or_published_socket_and_lock_refuse_without_repair() { for name in ["owner.json", "control.sock", "operation.lock"] { let fixture = Fixture::new(); fixture.dead(); + if name == "owner.json" { + // Publication3 carries no original owner-file inode. Its first + // private snapshot becomes the saved intent's later inode anchor. + fixture.store(&fixture.intent()); + } let path = fixture.owner_root().join(name); let original = if name == "owner.json" { Some(fs::read(&path).unwrap()) @@ -346,6 +351,31 @@ fn replaced_owner_socket_and_operation_lock_refuse_without_repair() { } } +#[test] +fn first_selection_captures_current_private_same_byte_owner_snapshot() { + let fixture = Fixture::new(); + fixture.dead(); + let path = fixture.owner_root().join("owner.json"); + let bytes = fs::read(&path).unwrap(); + let prior_id = id(&path).unwrap(); + fs::rename(&path, path.with_extension("retained")).unwrap(); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path) + .unwrap(); + file.write_all(&bytes).unwrap(); + file.sync_all().unwrap(); + assert_ne!(id(&path).unwrap(), prior_id); + let selected = select(&fixture.candidate, RUN).unwrap(); + assert_eq!(selected.owner_sha256, digest(&bytes)); + fixture.store(&fixture.intent()); + fs::remove_file(&path).unwrap(); + fs::rename(path.with_extension("retained"), &path).unwrap(); + fixture.assert_refused_unchanged(); +} + #[test] fn committed_intent_keeps_original_hash_and_inventory_through_cleanup_progress() { let fixture = Fixture::new(); From 055294f0b6392e82b987d9035fa566104b2b8ee0 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 13:51:32 -0400 Subject: [PATCH 07/47] fix(runtime): respect bounded recovery intent reader --- .../src/provider/graph/native/foreground/recovery.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs index b438274f1..8a502e711 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs @@ -9,7 +9,9 @@ use std::{ }; const FILE: &str = "live-owner-recovery.json"; -const LIMIT: usize = 128 * 1024; +// Keep the complete intent inside the existing secure reader's admitted bound. +// Oversized original selections cannot acquire retry authority. +const LIMIT: usize = 64 * 1024; fn refused() -> CandidateError { error( "native_graph_live_owner_recovery", From 74d01dd5a2397329216dfe2fb6fa1d53961f9593 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 13:55:57 -0400 Subject: [PATCH 08/47] fix(runtime): retain cleanup guards around engine calls --- packages/runtime-core/README.md | 7 ++ .../src/provider/graph/native/runtime.rs | 36 ++++++- .../provider/graph/native/runtime/tests.rs | 98 +++++++++++++++++++ .../src/provider/native_environment.rs | 39 ++++++-- 4 files changed, 173 insertions(+), 7 deletions(-) diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 14c9b9404..8fa7dc9d8 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -131,6 +131,13 @@ retains the original Ready selectors and exact inventory. Pending writes refuse; retired paths require the intent's preceding phase and unchanged archived inode. This library selector creates no intent, connects no provider and has no CLI or frontend recovery activation. + +Guarded native cleanup checks its retained authority before and after every +engine observation and stop/remove request, including failed requests. It also +checks each private-environment absence and retirement call. A lost guard refuses +before the next request or journal transition; an already attempted effect remains +durable evidence for an explicitly admitted retry. This prerequisite introduces +no dead-publisher cleanup entrypoint. Cancellation fences startup before admission and subsequent effects. Cleanup retains stop failures by admitted workload name, and retires the publication only after exact cleanup succeeds. Replies bind their requested action and immutable diff --git a/packages/runtime-core/src/provider/graph/native/runtime.rs b/packages/runtime-core/src/provider/graph/native/runtime.rs index dfd17dbca..9be5c7c22 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime.rs @@ -29,6 +29,35 @@ trait Backend { Err(refused()) } } +/// Recovery authority must survive every bounded engine call, including failed +/// observations. Losing it stops this attempt before another engine operation. +struct GuardedBackend<'a, B> { + backend: &'a B, + guard: Option<&'a dyn Fn() -> Result<(), CandidateError>>, +} +impl Backend for GuardedBackend<'_, B> { + fn request( + &self, + method: Method, + path: &str, + body: Option<&Value>, + ) -> Result { + check_startup(self.guard)?; + let result = self.backend.request(method, path, body); + check_startup(self.guard)?; + result + } + fn stop( + &self, + selected: &[(String, u64)], + admitted: &BTreeMap<&str, &str>, + ) -> Result<(), CandidateError> { + check_startup(self.guard)?; + let result = self.backend.stop(selected, admitted); + check_startup(self.guard)?; + result + } +} impl Backend for Engine<'_> { fn request( &self, @@ -631,6 +660,7 @@ pub(super) fn cleanup_guarded( ) -> Result { check_startup(guard)?; let engine = Engine::connect_cleanup(candidate)?; + check_startup(guard)?; let (mut receipt, root) = journal::load( candidate, run, @@ -640,6 +670,7 @@ pub(super) fn cleanup_guarded( if let Some(expected) = expected { receipt.check_binding(expected)?; } + check_startup(guard)?; let backend = OwnedBackend { engine, launcher: None, @@ -647,7 +678,8 @@ pub(super) fn cleanup_guarded( }; cleanup_using_guarded(&backend, &mut receipt, &root, guard)?; check_startup(guard)?; - native_environment::retire_graph(candidate, backend.engine.guest(), &receipt)?; + native_environment::retire_graph(candidate, backend.engine.guest(), &receipt, guard)?; + check_startup(guard)?; Ok(receipt) } #[cfg(test)] @@ -665,6 +697,8 @@ fn cleanup_using_guarded( guard: Option<&dyn Fn() -> Result<(), CandidateError>>, ) -> Result<(), CandidateError> { check_startup(guard)?; + let guarded = GuardedBackend { backend, guard }; + let backend = &guarded; // A committed cleanup phase is retry authority for this same inventory, never // permission to move the receipt back to startup or stop an already retired run. let stopped = matches!( diff --git a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs index 8b5dd27e1..7757d19b7 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs @@ -948,6 +948,104 @@ fn cleanup_owner_guard_refuses_before_provider_connection_or_receipt_mutation() assert_eq!(session.backend.receipt().phase, Phase::ReadyObserved); } +#[test] +fn cleanup_guard_loss_during_first_observation_prevents_next_request_and_stop_intent() { + struct Changed<'a> { + backend: &'a Fake, + changed: &'a Cell, + requests: Cell, + } + impl Backend for Changed<'_> { + fn request( + &self, + method: Method, + path: &str, + body: Option<&Value>, + ) -> Result { + self.requests.set(self.requests.get() + 1); + let result = self.backend.request(method, path, body); + self.changed.set(true); + result + } + } + let fixture = Fixture::new(basic()); + let (graph, mut session) = + fixture.session(fixture.prepared(json!({"web":{}}), &BTreeMap::new())); + execution::run(&graph, &mut session, Duration::from_secs(5)).unwrap(); + let changed = Cell::new(false); + let backend = Changed { + backend: &session.backend, + changed: &changed, + requests: Cell::new(0), + }; + let guard = || { + if changed.get() { + Err(error("native_graph_foreground", "changed")) + } else { + Ok(()) + } + }; + let before = fs::read(session.root.join("state.json")).unwrap(); + let effects = session.backend.state.borrow().effects.clone(); + assert_eq!( + cleanup_using_guarded(&backend, &mut session.receipt, &session.root, Some(&guard)) + .unwrap_err() + .code, + "native_graph_foreground" + ); + assert_eq!(backend.requests.get(), 1); + assert_eq!(session.backend.state.borrow().effects, effects); + assert_eq!(session.backend.state.borrow().stop_batches, 0); + assert_eq!(fs::read(session.root.join("state.json")).unwrap(), before); +} + +#[test] +fn cleanup_guard_rechecks_failed_stop_and_refuses_before_followup_request() { + struct FailedStop<'a>(&'a Cell); + impl Backend for FailedStop<'_> { + fn request( + &self, + _method: Method, + _path: &str, + _body: Option<&Value>, + ) -> Result { + panic!("lost guard cannot admit another engine request") + } + fn stop( + &self, + _selected: &[(String, u64)], + _admitted: &BTreeMap<&str, &str>, + ) -> Result<(), CandidateError> { + self.0.set(true); + Err(error("fake_stop_uncertain", "fixed")) + } + } + let changed = Cell::new(false); + let guard = || { + if changed.get() { + Err(error("native_graph_foreground", "changed")) + } else { + Ok(()) + } + }; + let backend = FailedStop(&changed); + let guarded = GuardedBackend { + backend: &backend, + guard: Some(&guard), + }; + assert_eq!( + guarded.stop(&[], &BTreeMap::new()).unwrap_err().code, + "native_graph_foreground" + ); + assert_eq!( + guarded + .request(Method::GET, "/unused", None) + .unwrap_err() + .code, + "native_graph_foreground" + ); +} + #[test] fn native_stop_failure_keeps_reservation_and_maps_only_owned_ids_to_workload_names() { let fixture = Fixture::new(basic()); diff --git a/packages/runtime-core/src/provider/native_environment.rs b/packages/runtime-core/src/provider/native_environment.rs index c949ef376..868fb8aa0 100644 --- a/packages/runtime-core/src/provider/native_environment.rs +++ b/packages/runtime-core/src/provider/native_environment.rs @@ -159,6 +159,24 @@ pub(super) fn retire( slot: &str, lease: Option<&EnvironmentLease>, ) -> Result<(), CandidateError> { + retire_guarded(candidate, guest, slot, lease, None) +} +fn check_guard( + guard: Option<&dyn Fn() -> Result<(), CandidateError>>, +) -> Result<(), CandidateError> { + if let Some(guard) = guard { + guard()?; + } + Ok(()) +} +fn retire_guarded( + candidate: &Candidate, + guest: &OwnedGuest<'_>, + slot: &str, + lease: Option<&EnvironmentLease>, + guard: Option<&dyn Fn() -> Result<(), CandidateError>>, +) -> Result<(), CandidateError> { + check_guard(guard)?; let intent = read(candidate, slot)?; if intent.incarnation != guest.incarnation() || intent.boot != guest.boot_id() @@ -183,10 +201,13 @@ pub(super) fn retire( false, )?; // A native payload may disappear only after the exact bound container name is absent. - super::engine::require_container_absent(guest, &intent.binding.container)?; - if guest.execute_cleanup(environment_recovery::RETIRE, &[slot, "same"])? - != "environment-removed-v1\n" - { + check_guard(guard)?; + let result = super::engine::require_container_absent(guest, &intent.binding.container); + check_guard(guard)?; + result?; + let result = guest.execute_cleanup(environment_recovery::RETIRE, &[slot, "same"]); + check_guard(guard)?; + if result? != "environment-removed-v1\n" { return Err(refused()); } Ok(()) @@ -195,7 +216,9 @@ pub(in crate::provider) fn retire_graph( candidate: &Candidate, guest: &OwnedGuest<'_>, receipt: &super::graph::native::Receipt, + guard: Option<&dyn Fn() -> Result<(), CandidateError>>, ) -> Result<(), CandidateError> { + check_guard(guard)?; let directory = root(candidate); if *receipt.phase() != super::graph::native::Phase::Removed { return Err(refused()); @@ -229,13 +252,17 @@ pub(in crate::provider) fn retire_graph( &intent.service, false, )?; - super::engine::require_container_absent(guest, &intent.binding.container)?; + check_guard(guard)?; + let result = super::engine::require_container_absent(guest, &intent.binding.container); + check_guard(guard)?; + result?; selected.push(slot.to_owned()); } } for slot in selected { - retire(candidate, guest, &slot, None)?; + retire_guarded(candidate, guest, &slot, None, guard)?; } + check_guard(guard)?; Ok(()) } From 9f425b5574997de1107d9f77d2e046e3eea5efc4 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 13:57:28 -0400 Subject: [PATCH 09/47] fix(runtime): guard cleanup journal publications --- packages/runtime-core/README.md | 4 +++- packages/runtime-core/src/provider/graph/native/runtime.rs | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 8fa7dc9d8..8e1fe4ead 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -133,7 +133,9 @@ This library selector creates no intent, connects no provider and has no CLI or frontend recovery activation. Guarded native cleanup checks its retained authority before and after every -engine observation and stop/remove request, including failed requests. It also +engine observation, removal request and admitted stop batch, including failed +calls. The existing stop batch retains its concurrent HTTP-stop semantics. Cleanup +checks authority before each journal publication. It also checks each private-environment absence and retirement call. A lost guard refuses before the next request or journal transition; an already attempted effect remains durable evidence for an explicitly admitted retry. This prerequisite introduces diff --git a/packages/runtime-core/src/provider/graph/native/runtime.rs b/packages/runtime-core/src/provider/graph/native/runtime.rs index 9be5c7c22..f05dadfc6 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime.rs @@ -746,6 +746,7 @@ fn cleanup_using_guarded( return Ok(()); } if !stopped { + check_startup(guard)?; receipt.phase = Phase::StopIntent; journal::save(root, receipt)?; } @@ -777,8 +778,10 @@ fn cleanup_using_guarded( } // A resumed removal still durably records each fresh terminal observation // before deletion, without moving its already committed phase backward. + check_startup(guard)?; journal::save(root, receipt)?; if !removing { + check_startup(guard)?; receipt.phase = Phase::RemovalIntent; journal::save(root, receipt)?; } @@ -799,6 +802,7 @@ fn cleanup_using_guarded( if inspected(backend, receipt, resource)?.is_some() { return Err(refused()); } + check_startup(guard)?; receipt.resources.get_mut(key).ok_or_else(refused)?.phase = "removed".into(); journal::save(root, receipt)?; } @@ -815,6 +819,7 @@ fn cleanup_using_guarded( { return Err(refused()); } + check_startup(guard)?; receipt .resources .get_mut("network:default") @@ -834,6 +839,7 @@ fn cleanup_using_guarded( return Err(refused()); } } + check_startup(guard)?; receipt .resources .get_mut("network:default") From 525d554022c28acdd5508d1f4e356a6903993eb6 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 14:02:44 -0400 Subject: [PATCH 10/47] feat(runtime): expose read-only native recovery selection --- packages/runtime-core/README.md | 9 +- packages/runtime-core/src/graph_cli/native.rs | 21 +++- .../graph/native/foreground/recovery/tests.rs | 100 +++++++++++++++++- .../tests/native_graph_cli_contract.rs | 46 ++++++++ 4 files changed, 171 insertions(+), 5 deletions(-) diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 8e1fe4ead..bc656d583 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -129,8 +129,13 @@ admission, not an external identity anchor from before that capture. After a durable intent, the saved inode and digest refuse replacement. Cleanup progress retains the original Ready selectors and exact inventory. Pending writes refuse; retired paths require the intent's preceding phase and unchanged archived inode. -This library selector creates no intent, connects no provider and has no CLI or -frontend recovery activation. +`graph native recovery-selection --run-id ID --json` exposes only the closed +selection: run, original value-free Ready receipt, raw receipt/owner hashes and +native host boot time. The receipt contains review hashes, workload readiness, +resource IDs/images/networks and terminal observations; it contains no environment +keys/values, command argv, source bytes or publication process/path metadata. The +selector creates no intent and connects no provider. It grants no cleanup or +frontend recovery authority. Guarded native cleanup checks its retained authority before and after every engine observation, removal request and admitted stop batch, including failed diff --git a/packages/runtime-core/src/graph_cli/native.rs b/packages/runtime-core/src/graph_cli/native.rs index 14476eea3..2bebc4365 100644 --- a/packages/runtime-core/src/graph_cli/native.rs +++ b/packages/runtime-core/src/graph_cli/native.rs @@ -9,7 +9,7 @@ use std::{ fn refused() -> CandidateError { CandidateError::new( "native_graph_arguments", - "Use graph native plan --source-file FILE, run|serve --source-file FILE --expect-review SHA [--environment-stdin] [--timeout-seconds 1..300], inspect|cleanup --run-id ID, or control --run-id ID --action status|cleanup; optional --json. Foreground serve/control require macOS. Native source and private envelopes require their exact kind/version; values omitted.", + "Use graph native plan --source-file FILE, run|serve --source-file FILE --expect-review SHA [--environment-stdin] [--timeout-seconds 1..300], inspect|cleanup|recovery-selection --run-id ID, or control --run-id ID --action status|cleanup; optional --json. Foreground serve/control and read-only recovery selection require macOS. Native source and private envelopes require their exact kind/version; values omitted.", ) } fn hex(value: &str, len: usize) -> bool { @@ -48,6 +48,25 @@ pub(super) fn command(candidate: &Candidate, args: &[&str]) -> Result return Err(refused()), } } + if *action == "recovery-selection" { + if private || singles.len() != 1 { + return Err(refused()); + } + let run = *singles + .get("--run-id") + .filter(|run| hex(run, 32)) + .ok_or_else(refused)?; + #[cfg(target_os = "macos")] + { + return serde_json::to_value(native::foreground::recovery::select(candidate, run)?) + .map_err(|_| refused()); + } + #[cfg(not(target_os = "macos"))] + { + let _ = run; + return Err(foreground_unavailable()); + } + } if *action == "control" { if private || singles.len() != 2 { return Err(refused()); diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs index 281a40208..3a14a5556 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs @@ -55,11 +55,23 @@ impl Fixture { } } fn ready(&self) -> Receipt { + let private = self.root.join("private-canary").exists(); let metadata: EnvMetadata = serde_json::from_value(json!({ "metadata_version":1,"overlay":null,"overlay_exists":false, - "workloads":{"web":{}},"inactive_scopes":[] + "workloads":{"web": if private {json!({"SELECTOR_SOURCE_KEY_CANARY":{"scope":"web","secret":true}})} else {json!({})}},"inactive_scopes":[] })) .unwrap(); + let values = if private { + BTreeMap::from([( + "web".into(), + BTreeMap::from([( + "SELECTOR_SOURCE_KEY_CANARY".into(), + "selector-private-value-canary".into(), + )]), + )]) + } else { + BTreeMap::new() + }; let prepared = selection::select( &self.candidate, selection::Options { @@ -73,9 +85,15 @@ impl Fixture { }, ) .unwrap() - .prepare(&self.candidate, &BTreeMap::new()) + .prepare(&self.candidate, &values) .unwrap(); let config = configuration(prepared.input(), OWNER).unwrap(); + if private { + let public = config.containers()["web"].to_string(); + assert!(public.contains("selector-argv-canary")); + assert!(public.contains("selector-literal-canary")); + assert!(!public.contains("selector-private-value-canary")); + } let mut receipt = Receipt::preparing(&config, OWNER, "12345678-abcd-abcd-abcd-123456789abc").unwrap(); receipt.phase = Phase::ReadyObserved; @@ -246,6 +264,84 @@ fn dead_complete_publication_selects_original_raw_hash_without_creating_authorit ); } +#[cfg(feature = "environment-launcher")] +#[test] +fn selection_json_excludes_real_compiler_argv_environment_keys_values_and_owner_bytes() { + let fixture = Fixture::new(); + fs::write(fixture.root.join("private-canary"), b"fixture").unwrap(); + fs::write( + fixture.root.join("project/.hack/hack.project.json"), + json!({ + "schema_version":1,"name":"selector","services":{"web":{ + "image":format!("sha256:{}", "d".repeat(64)), + "command":{"exec":["/bin/echo","selector-argv-canary","$EXACT"]}, + "environment":{ + "SELECTOR_LITERAL_KEY_CANARY":{"literal":"selector-literal-canary"}, + "SELECTOR_DEST_KEY_CANARY":{"env_ref":"SELECTOR_SOURCE_KEY_CANARY"} + } + }} + }) + .to_string(), + ) + .unwrap(); + fixture.dead(); + let intent = fixture.intent(); + assert!( + serde_json::to_string(&intent.publication) + .unwrap() + .contains(fixture.root.to_str().unwrap()) + ); + fixture.store(&intent); + let selected = select(&fixture.candidate, RUN).unwrap(); + let wire = serde_json::to_value(&selected).unwrap(); + let text = wire.to_string(); + for canary in [ + "selector-argv-canary", + "$EXACT", + "selector-literal-canary", + "selector-private-value-canary", + "SELECTOR_LITERAL_KEY_CANARY", + "SELECTOR_DEST_KEY_CANARY", + "SELECTOR_SOURCE_KEY_CANARY", + fixture.root.to_str().unwrap(), + "command", + "environment", + "process", + "publication", + "original", + ] { + assert!( + !text.contains(canary), + "selection leaked protected compiler/owner material" + ); + } + assert_eq!( + wire.as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect::>(), + [ + "host_boot_micros", + "kind", + "owner_sha256", + "receipt", + "receipt_sha256", + "run", + "version" + ] + ); + assert_eq!(wire["receipt"]["phase"], "ready-observed"); + assert_eq!( + wire["receipt"]["resources"]["container:web"]["id"], + selected.receipt.resources["container:web"] + .id + .as_deref() + .unwrap() + ); + assert!(!fixture.candidate.state_root.join("run/smolvm").exists()); +} + #[test] fn live_exact_publisher_never_yields_dead_recovery_authority() { let fixture = Fixture::new(); diff --git a/packages/runtime-core/tests/native_graph_cli_contract.rs b/packages/runtime-core/tests/native_graph_cli_contract.rs index 58389bcd6..d013327a4 100644 --- a/packages/runtime-core/tests/native_graph_cli_contract.rs +++ b/packages/runtime-core/tests/native_graph_cli_contract.rs @@ -420,6 +420,52 @@ fn public_native_control_requires_exact_action_and_never_adopts_absent_owner() { } } +#[test] +fn public_native_recovery_selection_is_closed_read_only_and_never_creates_missing_authority() { + let fixture = Fixture::new(); + let run = "b".repeat(32); + let output = fixture.invoke(&[ + "graph", + "native", + "recovery-selection", + "--run-id", + &run, + "--json", + ]); + assert!(!output.status.success()); + #[cfg(target_os = "macos")] + assert!(String::from_utf8_lossy(&output.stderr).contains("native_graph_live_owner_recovery")); + #[cfg(not(target_os = "macos"))] + assert!( + String::from_utf8_lossy(&output.stderr).contains("native_graph_foreground_unsupported") + ); + fixture.assert_no_state(); + for extra in [ + vec!["--environment-stdin"], + vec!["--source-file", fixture.source_path()], + vec!["--action", "cleanup"], + vec!["--expect-review", "private-selection-canary"], + vec!["--timeout-seconds", "1"], + vec!["--run-id", &run], + ] { + let mut args = vec![ + "graph", + "native", + "recovery-selection", + "--run-id", + &run, + "--json", + ]; + args.extend(extra); + let output = fixture.invoke(&args); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("native_graph_arguments")); + assert!(!String::from_utf8_lossy(&output.stderr).contains("private-selection-canary")); + assert!(!String::from_utf8_lossy(&output.stdout).contains("private-selection-canary")); + fixture.assert_no_state(); + } +} + #[cfg(not(target_os = "macos"))] #[test] fn public_native_serve_refuses_platform_before_private_descriptor_or_source_read() { From 7470a5de8ad65fa6411b66b355f938a85c08d068 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 14:09:14 -0400 Subject: [PATCH 11/47] test(runtime): bind privacy and lost-guard evidence precisely --- .../graph/native/foreground/recovery/tests.rs | 17 ++++++++++++----- .../src/provider/graph/native/runtime/tests.rs | 14 +++++++++++++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs index 3a14a5556..8ce30dfe6 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs @@ -304,11 +304,12 @@ fn selection_json_excludes_real_compiler_argv_environment_keys_values_and_owner_ "SELECTOR_DEST_KEY_CANARY", "SELECTOR_SOURCE_KEY_CANARY", fixture.root.to_str().unwrap(), - "command", - "environment", - "process", - "publication", - "original", + "\"command\":", + "\"environment\":", + "\"argv\":", + "\"process\":", + "\"publication\":", + "\"original\":", ] { assert!( !text.contains(canary), @@ -332,6 +333,12 @@ fn selection_json_excludes_real_compiler_argv_environment_keys_values_and_owner_ ] ); assert_eq!(wire["receipt"]["phase"], "ready-observed"); + assert!(super::super::super::super::hex( + wire["receipt"]["review"]["provenance"]["input"]["environment_policy_hash"] + .as_str() + .unwrap(), + 64 + )); assert_eq!( wire["receipt"]["resources"]["container:web"]["id"], selected.receipt.resources["container:web"] diff --git a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs index 7757d19b7..92a46bd0a 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime/tests.rs @@ -1126,13 +1126,25 @@ fn cleanup_owner_guard_fences_delete_after_stop_and_network_delete_after_contain after_delete ); let (mut retained, _) = journal::load(&fixture.candidate, RUN, OWNER, BOOT).unwrap(); - assert!(retained.terminal["container:web"].stop_requested); + if after_delete { + assert_eq!(retained.phase, Phase::RemovalIntent); + assert!(retained.terminal["container:web"].stop_requested); + } else { + // Authority was lost inside the admitted stop batch: its result is + // retained as uncertain, with no unauthorized terminal observation. + assert_eq!(retained.phase, Phase::StopIntent); + assert!(retained.terminal.is_empty()); + } changed.set(false); session.backend.state.borrow_mut().cancel_after_stop = None; session.backend.state.borrow_mut().cancel_after_delete = None; cleanup_using_guarded(&session.backend, &mut retained, &session.root, Some(&guard)) .unwrap(); assert_eq!(retained.phase, Phase::Removed); + assert_eq!( + retained.terminal["container:web"].stop_requested, + after_delete + ); let effects = &session.backend.state.borrow().effects; assert_eq!( effects From 9dc79b792f4d7ccd54268583f36d036bd7f4acf6 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 14:21:44 -0400 Subject: [PATCH 12/47] feat(runtime): recover exact same-boot native publications --- packages/runtime-core/README.md | 10 + .../provider/graph/native/foreground/owner.rs | 36 +++ .../graph/native/foreground/recovery.rs | 61 +++- .../native/foreground/recovery/cleanup.rs | 269 ++++++++++++++++++ .../graph/native/foreground/recovery/tests.rs | 256 +++++++++++++++++ .../src/provider/graph/native/runtime.rs | 50 +++- .../src/provider/native_environment.rs | 38 ++- 7 files changed, 706 insertions(+), 14 deletions(-) create mode 100644 packages/runtime-core/src/provider/graph/native/foreground/recovery/cleanup.rs diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index bc656d583..2e7240abb 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -137,6 +137,16 @@ keys/values, command argv, source bytes or publication process/path metadata. Th selector creates no intent and connects no provider. It grants no cleanup or frontend recovery authority. +The separate explicit library recovery entrypoint re-admits both raw selectors +and commits a bounded 64 KiB recovery intent before cleanup. It retains the +publication gate, original operation lock and cleanup provider lease through +matching Removed resources, private-environment absence and publication +retirement. Durable phases precede each exclusive archive move; original Ready +hashes and inventory remain fixed while cleanup phases advance. Completed retry +checks absence rather than repeating retirement. Pending writes, unexpected +replacements or missing original and archive paths retain a refusal. Public +frontend recovery and actual dead-owner acceptance remain separate gates. + Guarded native cleanup checks its retained authority before and after every engine observation, removal request and admitted stop batch, including failed calls. The existing stop batch retains its concurrent HTTP-stop semantics. Cleanup diff --git a/packages/runtime-core/src/provider/graph/native/foreground/owner.rs b/packages/runtime-core/src/provider/graph/native/foreground/owner.rs index c75c9c394..0d1431a44 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/owner.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/owner.rs @@ -364,6 +364,30 @@ impl<'a> RecoveryLease<'a> { .record(self.candidate, self.run) .map(|(_, boot)| boot) } + /// Caller has already durably admitted this exact path's retirement phase. + /// Existing archive identity is retry evidence, never overwrite permission. + pub(super) fn archive(&self, socket: bool) -> Result<(), CandidateError> { + let (socket_original, owner_original) = self.verify(true, !socket)?; + if !socket && socket_original { + return Err(refused()); + } + let original = if socket { + socket_original + } else { + owner_original + }; + if original { + let path = self + .root + .join(if socket { "control.sock" } else { "owner.json" }); + exclusive_move(&path, &self.archive_path(socket))?; + fs::File::open(&self.root) + .and_then(|file| file.sync_all()) + .map_err(|_| refused())?; + } + self.verify(true, !socket)?; + Ok(()) + } fn archive_path(&self, socket: bool) -> PathBuf { let fingerprint = self.selection.fingerprint(); self.root.join(if socket { @@ -437,6 +461,18 @@ impl<'a> RecoveryLease<'a> { } } +pub(super) fn exclusive_move(source: &Path, target: &Path) -> Result<(), CandidateError> { + use std::{ffi::CString, os::unix::ffi::OsStrExt}; + let source = CString::new(source.as_os_str().as_bytes()).map_err(|_| refused())?; + let target = CString::new(target.as_os_str().as_bytes()).map_err(|_| refused())?; + // SAFETY: both paths are NUL-terminated for this synchronous call. RENAME_EXCL + // refuses any existing target; no replacement or copy fallback is permitted. + if unsafe { libc::renamex_np(source.as_ptr(), target.as_ptr(), libc::RENAME_EXCL) } != 0 { + return Err(refused()); + } + Ok(()) +} + pub(super) struct Publication { listener: UnixListener, pin: Pin, diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs index 8a502e711..85fd4fb74 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs @@ -74,12 +74,12 @@ impl Progress { Ok(()) } } -#[derive(Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] enum IntentKind { NativeGraphLiveOwnerRecovery, } -#[derive(Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct Intent { version: u8, @@ -183,10 +183,38 @@ pub struct Selection { owner_sha256: String, host_boot_micros: u64, } +struct Admitted<'a> { + candidate: &'a Candidate, + run: &'a str, + root: PathBuf, + root_identity: (u64, u64), + lease: owner::RecoveryLease<'a>, + original: Receipt, + original_bytes: String, + receipt_sha256: String, + intent: Option, + witness: Option<((u64, u64), Vec)>, +} +impl Admitted<'_> { + fn selection(&self) -> Result { + Ok(Selection { + version: 1, + kind: SelectionKind::NativeGraphRecoverySelection, + run: self.run.into(), + receipt: self.original.clone(), + receipt_sha256: self.receipt_sha256.clone(), + owner_sha256: self.lease.selected().fingerprint(), + host_boot_micros: self.lease.host_boot_micros().map_err(|_| refused())?, + }) + } +} /// Select only a dead version3 complete Ready publication, or its unchanged /// committed recovery intent. No file is created or repaired, and no provider /// operation, private acquisition, signal or cleanup is performed. pub fn select(candidate: &Candidate, run: &str) -> Result { + admit(candidate, run)?.selection() +} +fn admit<'a>(candidate: &'a Candidate, run: &'a str) -> Result, CandidateError> { let root = directory(candidate, run)?; let root_identity = id(&root)?; let intent_path = root.join(FILE); @@ -240,11 +268,11 @@ pub fn select(candidate: &Candidate, run: &str) -> Result {} + == *original_bytes => {} None if !exists(&intent_path)? => {} _ => return Err(refused()), } @@ -254,16 +282,25 @@ pub fn select(candidate: &Candidate, run: &str) -> Result)>, +} +pub(super) struct Context<'a> { + admitted: Admitted<'a>, + committed: RefCell, +} +impl<'a> Context<'a> { + fn new(mut admitted: Admitted<'a>) -> Result { + let intent = match admitted.intent.take() { + Some(intent) => intent, + None => Intent { + version: 1, + kind: IntentKind::NativeGraphLiveOwnerRecovery, + run: admitted.run.into(), + journal_parent: admitted.root_identity, + original: admitted.original_bytes.clone(), + receipt_sha256: admitted.receipt_sha256.clone(), + publication: admitted.lease.selected().clone(), + progress: Progress::Cleanup, + receipt_progress: 0, + resource_progress: resource_progress(&admitted.original)?, + }, + }; + let witness = admitted.witness.take(); + Ok(Self { + admitted, + committed: RefCell::new(Committed { intent, witness }), + }) + } + fn verify(&self, pending: Option<((u64, u64), &[u8])>) -> Result { + let admitted = &self.admitted; + for path in admitted + .root + .ancestors() + .take_while(|path| path.starts_with(&admitted.candidate.state_root)) + { + state::check_private_directory(path).map_err(|_| refused())?; + } + if id(&admitted.root)? != admitted.root_identity + || exists(&admitted.root.join("state.pending"))? + { + return Err(refused()); + } + let pending_path = admitted.root.join("live-owner-recovery.pending"); + match pending { + Some((identity, bytes)) + if id(&pending_path)? == identity + && native_input::read_file(&pending_path, LIMIT).map_err(|_| refused())? + == bytes => {} + None if !exists(&pending_path)? => {} + _ => return Err(refused()), + } + let committed = self.committed.borrow(); + let path = admitted.root.join(FILE); + match &committed.witness { + Some((identity, bytes)) + if id(&path)? == *identity + && native_input::read_file(&path, LIMIT).map_err(|_| refused())? == *bytes => {} + None if !exists(&path)? => {} + _ => return Err(refused()), + } + committed.intent.progress.verify(&admitted.lease)?; + let (current, root, bytes) = + journal::read_recovery(admitted.candidate, admitted.original.review())?; + committed + .intent + .current(&root, &admitted.original, ¤t, &bytes)?; + if committed.witness.is_none() && bytes != admitted.original_bytes { + return Err(refused()); + } + committed.intent.progress.verify(&admitted.lease)?; + Ok(current) + } + fn publish(&self, next: Intent) -> Result<(), CandidateError> { + let current = self.verify(None)?; + next.current( + &self.admitted.root, + &self.admitted.original, + ¤t, + &journal::read_recovery(self.admitted.candidate, self.admitted.original.review())?.2, + )?; + let bytes = serde_json::to_vec_pretty(&next).map_err(|_| refused())?; + if bytes.is_empty() || bytes.len() > LIMIT { + return Err(refused()); + } + let path = self.admitted.root.join(FILE); + let pending = self.admitted.root.join("live-owner-recovery.pending"); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW) + .open(&pending) + .map_err(|_| refused())?; + let identity = ( + file.metadata().map_err(|_| refused())?.dev(), + file.metadata().map_err(|_| refused())?.ino(), + ); + file.write_all(&bytes) + .and_then(|()| file.sync_all()) + .map_err(|_| refused())?; + self.verify(Some((identity, &bytes)))?; + if self.committed.borrow().witness.is_some() { + fs::rename(&pending, &path).map_err(|_| refused())?; + } else { + owner::exclusive_move(&pending, &path).map_err(|_| refused())?; + } + File::open(&self.admitted.root) + .and_then(|file| file.sync_all()) + .map_err(|_| refused())?; + if id(&path)? != identity + || native_input::read_file(&path, LIMIT).map_err(|_| refused())? != bytes + { + return Err(refused()); + } + *self.committed.borrow_mut() = Committed { + intent: next, + witness: Some((identity, bytes)), + }; + self.verify(None)?; + Ok(()) + } + pub(super) fn guard(&self) -> Result<(), CandidateError> { + let current = self.verify(None)?; + let mut next = self.committed.borrow().intent.clone(); + let rank = receipt_progress(¤t)?; + let resources = resource_progress(¤t)?; + if next.receipt_progress != rank || next.resource_progress != resources { + next.receipt_progress = rank; + next.resource_progress = resources; + self.publish(next)?; + } + Ok(()) + } + fn advance(&self, progress: Progress) -> Result<(), CandidateError> { + self.guard()?; + let mut next = self.committed.borrow().intent.clone(); + if progress < next.progress { + return Err(refused()); + } + if progress > next.progress { + next.progress = progress; + self.publish(next)?; + } + Ok(()) + } + fn finish(&self, snapshot: &runtime::Snapshot) -> Result<(), CandidateError> { + self.finish_using(snapshot, &|_| Ok(())) + } + pub(super) fn finish_using( + &self, + snapshot: &runtime::Snapshot, + checkpoint: &dyn Fn(Progress) -> Result<(), CandidateError>, + ) -> Result<(), CandidateError> { + snapshot + .receipt + .check_binding(&self.admitted.original) + .map_err(|_| refused())?; + if snapshot.receipt.phase != Phase::Removed + || snapshot + .observations + .keys() + .ne(snapshot.receipt.readiness.keys()) + || snapshot.observations.values().any(Option::is_some) + { + return Err(refused()); + } + if self.committed.borrow().intent.progress <= Progress::EnvironmentRetired { + self.advance(Progress::EnvironmentRetired)?; + } + if self.committed.borrow().intent.progress <= Progress::SocketRetirementIntent { + self.advance(Progress::SocketRetirementIntent)?; + checkpoint(Progress::SocketRetirementIntent)?; + self.admitted.lease.archive(true)?; + self.advance(Progress::SocketRetired)?; + checkpoint(Progress::SocketRetired)?; + } + if self.committed.borrow().intent.progress <= Progress::OwnerRetirementIntent { + self.advance(Progress::OwnerRetirementIntent)?; + checkpoint(Progress::OwnerRetirementIntent)?; + self.admitted.lease.archive(false)?; + self.advance(Progress::Complete)?; + } + self.guard() + } +} + +/// Exact original selectors are independently re-admitted; no source replay, +/// environment acquisition, startup, reboot or data removal is available. +pub struct Options<'a> { + pub run: &'a str, + pub expect_receipt: &'a str, + pub expect_owner: &'a str, +} +#[derive(Serialize)] +#[serde(rename_all = "kebab-case")] +enum ResultKind { + NativeGraphLiveOwnerRecovered, +} +#[derive(Serialize)] +pub struct Outcome { + version: u8, + kind: ResultKind, + run: String, + same_boot: bool, + publication_retired: bool, + receipt: Receipt, +} +pub fn recover(candidate: &Candidate, options: Options<'_>) -> Result { + recover_using(candidate, options, |context, environment_retired| { + runtime::cleanup_recovery( + candidate, + context.admitted.run, + &context.admitted.original, + &|| context.guard(), + environment_retired, + &|snapshot| context.finish(snapshot), + ) + }) +} +pub(super) fn recover_using( + candidate: &Candidate, + options: Options<'_>, + cleanup: impl FnOnce(&Context<'_>, bool) -> Result, +) -> Result { + let context = Context::new(admit(candidate, options.run)?)?; + if !super::super::super::super::hex(options.expect_receipt, 64) + || !super::super::super::super::hex(options.expect_owner, 64) + || context.admitted.receipt_sha256 != options.expect_receipt + || context.admitted.lease.selected().fingerprint() != options.expect_owner + { + return Err(refused()); + } + if context.committed.borrow().witness.is_none() { + let intent = context.committed.borrow().intent.clone(); + context.publish(intent)?; + } + let environment_retired = + context.committed.borrow().intent.progress >= Progress::EnvironmentRetired; + let receipt = cleanup(&context, environment_retired)?; + context.guard()?; + let current = context.verify(None)?; + if context.committed.borrow().intent.progress != Progress::Complete + || receipt.phase != Phase::Removed + || serde_json::to_vec(&receipt).map_err(|_| refused())? + != serde_json::to_vec(¤t).map_err(|_| refused())? + { + return Err(refused()); + } + Ok(Outcome { + version: 1, + kind: ResultKind::NativeGraphLiveOwnerRecovered, + run: options.run.into(), + same_boot: true, + publication_retired: true, + receipt, + }) +} diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs index 8ce30dfe6..ec326041b 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs @@ -579,3 +579,259 @@ fn retired_publication_paths_require_prior_exact_phase_and_never_missing_both() fs::remove_file(socket_archive).unwrap(); assert!(select(&fixture.candidate, RUN).is_err()); } + +fn recovery_options(selected: &Selection) -> Options<'_> { + Options { + run: RUN, + expect_receipt: &selected.receipt_sha256, + expect_owner: &selected.owner_sha256, + } +} +fn removed(fixture: &Fixture) -> Receipt { + let mut receipt = fixture.receipt(); + receipt.phase = Phase::Removed; + for resource in receipt.resources.values_mut() { + resource.phase = "removed".into(); + } + journal::save(&fixture.journal_root(), &receipt).unwrap(); + receipt +} +fn absent_snapshot(receipt: Receipt) -> runtime::Snapshot { + runtime::Snapshot { + observations: receipt + .readiness + .keys() + .map(|key| (key.clone(), None)) + .collect(), + receipt, + } +} + +#[test] +fn effect_recovery_requires_exact_selectors_before_intent_or_driver_entry() { + let fixture = Fixture::new(); + fixture.dead(); + let selected = select(&fixture.candidate, RUN).unwrap(); + let called = std::cell::Cell::new(0); + for owner in [false, true] { + let mut options = recovery_options(&selected); + let wrong = "f".repeat(64); + if owner { + options.expect_owner = &wrong; + } else { + options.expect_receipt = &wrong; + } + assert!( + cleanup::recover_using(&fixture.candidate, options, |_, _| { + called.set(called.get() + 1); + panic!("wrong original selection cannot reach cleanup") + }) + .is_err() + ); + assert!(!fixture.journal_root().join(FILE).exists()); + let again = select(&fixture.candidate, RUN).unwrap(); + assert_eq!(again.receipt_sha256, selected.receipt_sha256); + assert_eq!(again.owner_sha256, selected.owner_sha256); + } + assert_eq!(called.get(), 0); +} + +#[test] +fn recovery_commits_original_before_driver_and_archives_only_after_removed_absence() { + let fixture = Fixture::new(); + fixture.dead(); + let selected = select(&fixture.candidate, RUN).unwrap(); + let lock = id(&fixture.owner_root().join("operation.lock")).unwrap(); + let outcome = cleanup::recover_using( + &fixture.candidate, + recovery_options(&selected), + |context, retired| { + assert!(!retired); + let intent: Intent = serde_json::from_slice( + &native_input::read_file(&fixture.journal_root().join(FILE), LIMIT).unwrap(), + ) + .unwrap(); + assert_eq!(intent.receipt_sha256, selected.receipt_sha256); + assert_eq!(intent.publication.fingerprint(), selected.owner_sha256); + assert!(intent.progress == Progress::Cleanup); + context.guard()?; + for phase in [Phase::StopIntent, Phase::Stopped, Phase::RemovalIntent] { + let mut receipt = fixture.receipt(); + receipt.phase = phase; + journal::save(&fixture.journal_root(), &receipt)?; + context.guard()?; + } + let receipt = removed(&fixture); + context.finish_using(&absent_snapshot(receipt.clone()), &|_| Ok(()))?; + Ok(receipt) + }, + ) + .unwrap(); + let wire = serde_json::to_value(outcome).unwrap(); + assert_eq!(wire["publication_retired"], true); + assert_eq!(wire["receipt"]["phase"], "removed"); + assert!(!fixture.owner_root().join("owner.json").exists()); + assert!(!fixture.owner_root().join("control.sock").exists()); + assert_eq!( + id(&fixture.owner_root().join("operation.lock")).unwrap(), + lock + ); + let again = select(&fixture.candidate, RUN).unwrap(); + assert_eq!(again.receipt_sha256, selected.receipt_sha256); + assert_eq!(again.owner_sha256, selected.owner_sha256); + assert_eq!(again.receipt.phase, Phase::ReadyObserved); + let intent = fs::read(fixture.journal_root().join(FILE)).unwrap(); + cleanup::recover_using( + &fixture.candidate, + recovery_options(&selected), + |context, retired| { + assert!(retired); + let receipt = fixture.receipt(); + context.finish_using(&absent_snapshot(receipt.clone()), &|_| { + panic!("complete retry cannot repeat archive") + })?; + Ok(receipt) + }, + ) + .unwrap(); + assert_eq!(fs::read(fixture.journal_root().join(FILE)).unwrap(), intent); + assert!(!fixture.candidate.state_root.join("run/smolvm").exists()); +} + +#[test] +fn publication_retirement_interruption_retries_only_prior_exact_phase_and_inventory() { + for stop in [ + Progress::SocketRetirementIntent, + Progress::SocketRetired, + Progress::OwnerRetirementIntent, + ] { + let fixture = Fixture::new(); + fixture.dead(); + let selected = select(&fixture.candidate, RUN).unwrap(); + let lock = id(&fixture.owner_root().join("operation.lock")).unwrap(); + assert!( + cleanup::recover_using( + &fixture.candidate, + recovery_options(&selected), + |context, _| { + let receipt = removed(&fixture); + context.finish_using(&absent_snapshot(receipt), &|phase| { + if phase == stop { + Err(refused()) + } else { + Ok(()) + } + })?; + panic!("injected durable retirement boundary must interrupt") + } + ) + .is_err() + ); + let before = select(&fixture.candidate, RUN).unwrap(); + assert_eq!(before.receipt_sha256, selected.receipt_sha256); + cleanup::recover_using( + &fixture.candidate, + recovery_options(&selected), + |context, retired| { + assert!(retired); + let receipt = fixture.receipt(); + context.finish_using(&absent_snapshot(receipt.clone()), &|_| Ok(()))?; + Ok(receipt) + }, + ) + .unwrap(); + assert_eq!( + id(&fixture.owner_root().join("operation.lock")).unwrap(), + lock + ); + assert!(!fixture.owner_root().join("owner.json").exists()); + assert!(!fixture.owner_root().join("control.sock").exists()); + } +} + +#[test] +fn oversized_intent_refuses_before_pending_publication_or_cleanup() { + let fixture = Fixture::new(); + fixture.dead(); + let path = fixture.journal_root().join("state.json"); + let original = fs::read_to_string(&path).unwrap(); + let padded = format!("{}{}", original, " ".repeat(LIMIT - original.len())); + fs::write(&path, &padded).unwrap(); + let selected = select(&fixture.candidate, RUN).unwrap(); + let called = std::cell::Cell::new(false); + assert!( + cleanup::recover_using(&fixture.candidate, recovery_options(&selected), |_, _| { + called.set(true); + panic!("oversized serialized intent cannot reach cleanup") + }) + .is_err() + ); + assert!(!called.get()); + assert!(!fixture.journal_root().join(FILE).exists()); + assert!( + !fixture + .journal_root() + .join("live-owner-recovery.pending") + .exists() + ); + assert_eq!(fs::read_to_string(path).unwrap(), padded); +} + +#[test] +fn fresh_absence_and_unchanged_publication_are_required_before_retirement() { + for substituted in [false, true] { + let fixture = Fixture::new(); + fixture.dead(); + let selected = select(&fixture.candidate, RUN).unwrap(); + let owner = fs::read(fixture.owner_root().join("owner.json")).unwrap(); + assert!( + cleanup::recover_using( + &fixture.candidate, + recovery_options(&selected), + |context, _| { + let receipt = removed(&fixture); + let mut observed = absent_snapshot(receipt); + if !substituted { + observed.observations.insert( + "web".into(), + Some(Observation::Running { + health: execution::Health::None, + }), + ); + } + context.finish_using(&observed, &|phase| { + if substituted && phase == Progress::SocketRetirementIntent { + let path = fixture.owner_root().join("control.sock"); + fs::rename(&path, path.with_extension("retained")).unwrap(); + let _replacement = + std::os::unix::net::UnixListener::bind(&path).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + } + Ok(()) + })?; + panic!("unproved absence or substituted socket cannot complete retirement") + } + ) + .is_err() + ); + assert_eq!( + fs::read(fixture.owner_root().join("owner.json")).unwrap(), + owner + ); + assert!(fixture.owner_root().join("control.sock").exists()); + assert!(fixture.journal_root().join(FILE).exists()); + let prefix = &selected.owner_sha256[..24]; + assert!( + !fixture + .owner_root() + .join(format!("owner-{prefix}.retired.json")) + .exists() + ); + assert!( + !fixture + .owner_root() + .join(format!("control-{prefix}.retired.sock")) + .exists() + ); + } +} diff --git a/packages/runtime-core/src/provider/graph/native/runtime.rs b/packages/runtime-core/src/provider/graph/native/runtime.rs index f05dadfc6..9476862ed 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime.rs @@ -657,6 +657,34 @@ pub(super) fn cleanup_guarded( run: &str, expected: Option<&Receipt>, guard: Option<&dyn Fn() -> Result<(), CandidateError>>, +) -> Result { + cleanup_inner(candidate, run, expected, guard, false, None) +} +#[cfg(target_os = "macos")] +pub(super) fn cleanup_recovery( + candidate: &Candidate, + run: &str, + expected: &Receipt, + guard: &dyn Fn() -> Result<(), CandidateError>, + environment_retired: bool, + finish: &dyn Fn(&Snapshot) -> Result<(), CandidateError>, +) -> Result { + cleanup_inner( + candidate, + run, + Some(expected), + Some(guard), + environment_retired, + Some(finish), + ) +} +fn cleanup_inner( + candidate: &Candidate, + run: &str, + expected: Option<&Receipt>, + guard: Option<&dyn Fn() -> Result<(), CandidateError>>, + environment_retired: bool, + finish: Option<&dyn Fn(&Snapshot) -> Result<(), CandidateError>>, ) -> Result { check_startup(guard)?; let engine = Engine::connect_cleanup(candidate)?; @@ -678,8 +706,28 @@ pub(super) fn cleanup_guarded( }; cleanup_using_guarded(&backend, &mut receipt, &root, guard)?; check_startup(guard)?; - native_environment::retire_graph(candidate, backend.engine.guest(), &receipt, guard)?; + if !environment_retired { + native_environment::retire_graph(candidate, backend.engine.guest(), &receipt, guard)?; + } check_startup(guard)?; + if let Some(finish) = finish { + // Keep the cleanup provider lease through fresh absence and publication + // retirement; no second startup or cleanup connection supplies authority. + native_environment::verify_retired_graph( + candidate, + backend.engine.guest(), + &receipt, + guard, + )?; + let guarded = GuardedBackend { + backend: &backend, + guard, + }; + let observed = snapshot(&guarded, receipt.clone())?; + check_startup(guard)?; + finish(&observed)?; + check_startup(guard)?; + } Ok(receipt) } #[cfg(test)] diff --git a/packages/runtime-core/src/provider/native_environment.rs b/packages/runtime-core/src/provider/native_environment.rs index 868fb8aa0..1f02f8f2b 100644 --- a/packages/runtime-core/src/provider/native_environment.rs +++ b/packages/runtime-core/src/provider/native_environment.rs @@ -217,6 +217,23 @@ pub(in crate::provider) fn retire_graph( guest: &OwnedGuest<'_>, receipt: &super::graph::native::Receipt, guard: Option<&dyn Fn() -> Result<(), CandidateError>>, +) -> Result<(), CandidateError> { + graph_retirement(candidate, guest, receipt, guard, false) +} +pub(in crate::provider) fn verify_retired_graph( + candidate: &Candidate, + guest: &OwnedGuest<'_>, + receipt: &super::graph::native::Receipt, + guard: Option<&dyn Fn() -> Result<(), CandidateError>>, +) -> Result<(), CandidateError> { + graph_retirement(candidate, guest, receipt, guard, true) +} +fn graph_retirement( + candidate: &Candidate, + guest: &OwnedGuest<'_>, + receipt: &super::graph::native::Receipt, + guard: Option<&dyn Fn() -> Result<(), CandidateError>>, + already_retired: bool, ) -> Result<(), CandidateError> { check_guard(guard)?; let directory = root(candidate); @@ -260,11 +277,30 @@ pub(in crate::provider) fn retire_graph( } } for slot in selected { - retire_guarded(candidate, guest, &slot, None, guard)?; + if already_retired { + check_guard(guard)?; + let result = guest.execute_cleanup(VERIFY_REMOVED, &[&slot]); + check_guard(guard)?; + if result? != "environment-removed-v1\n" { + return Err(refused()); + } + } else { + retire_guarded(candidate, guest, &slot, None, guard)?; + } } check_guard(guard)?; Ok(()) } +const VERIFY_REMOVED: &str = r#" +( +set -eu +root="/run/$1" +test ! -L "$root" +test ! -e "$root" +) >/dev/null 2>&1 +printf 'environment-removed-v1\n' +"#; + #[cfg(test)] mod tests; From 113b4dabf737813f49996825af48e9a4f754ecd3 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 14:34:43 -0400 Subject: [PATCH 13/47] fix(runtime): bind native recovery environment completeness --- packages/runtime-core/README.md | 7 +- .../src/provider/environment_recovery.rs | 22 ++ .../graph/native/foreground/recovery.rs | 2 + .../native/foreground/recovery/cleanup.rs | 10 + .../graph/native/foreground/recovery/tests.rs | 75 +++++++ .../src/provider/graph/native/journal.rs | 4 + .../src/provider/graph/native/runtime.rs | 29 ++- .../src/provider/native_environment.rs | 195 +++++++++++++++++- .../src/provider/native_environment/tests.rs | 92 +++++++++ 9 files changed, 433 insertions(+), 3 deletions(-) diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 2e7240abb..cbb4acecb 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -141,7 +141,12 @@ The separate explicit library recovery entrypoint re-admits both raw selectors and commits a bounded 64 KiB recovery intent before cleanup. It retains the publication gate, original operation lock and cleanup provider lease through matching Removed resources, private-environment absence and publication -retirement. Durable phases precede each exclusive archive move; original Ready +retirement. A metadata-only current slot inventory is bound into the intent; +changed or missing records refuse. Bounded read-only guest slot enumeration +cross-checks the validated native and Compose host records before cleanup and +again before publication retirement. Unknown guest slots refuse even when host +records are empty; no payload contents are read or inferred from missing records. +Durable phases precede each exclusive archive move; original Ready hashes and inventory remain fixed while cleanup phases advance. Completed retry checks absence rather than repeating retirement. Pending writes, unexpected replacements or missing original and archive paths retain a refusal. Public diff --git a/packages/runtime-core/src/provider/environment_recovery.rs b/packages/runtime-core/src/provider/environment_recovery.rs index cc4283ae4..6ec1ea917 100644 --- a/packages/runtime-core/src/provider/environment_recovery.rs +++ b/packages/runtime-core/src/provider/environment_recovery.rs @@ -259,6 +259,28 @@ pub fn recorded_slots(candidate: &Candidate) -> Result, CandidateErr } Ok(slots.into_iter().collect()) } +/// Read-only current-guest metadata for native recovery completeness. Pending +/// records refuse rather than being promoted or used as missing-slot authority. +#[cfg(all(target_os = "macos", feature = "native-config-plan"))] +pub(super) fn known_current_slots( + candidate: &Candidate, + incarnation: &str, + boot: &str, +) -> Result, CandidateError> { + let mut known = std::collections::BTreeSet::new(); + for slot in recorded_slots(candidate)? { + let path = root(candidate).join(format!("{slot}.pending")); + match fs::symlink_metadata(path) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + _ => return Err(error()), + } + let intent = read_mode(candidate, &slot, incarnation, None, false)?; + if intent.boot == boot { + known.insert(slot); + } + } + Ok(known) +} /// Validates immutable records before associating them with graph cleanup. pub(super) fn graph_slots( candidate: &Candidate, diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs index 85fd4fb74..3bf34dbf1 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs @@ -92,6 +92,7 @@ struct Intent { progress: Progress, receipt_progress: u8, resource_progress: BTreeMap, + environment: crate::provider::native_environment::Inventory, } fn receipt_progress(receipt: &Receipt) -> Result { match receipt.phase { @@ -279,6 +280,7 @@ fn admit<'a>(candidate: &'a Candidate, run: &'a str) -> Result, Can if let Some(intent) = &intent { intent.current(&root, &original, &again, ¤t_bytes)?; intent.progress.verify(&lease)?; + intent.environment.verify(candidate, &original)?; } else { Progress::Cleanup.verify(&lease)?; } diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery/cleanup.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery/cleanup.rs index 06f8bae3c..0c33345c6 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/recovery/cleanup.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery/cleanup.rs @@ -30,6 +30,10 @@ impl<'a> Context<'a> { progress: Progress::Cleanup, receipt_progress: 0, resource_progress: resource_progress(&admitted.original)?, + environment: crate::provider::native_environment::Inventory::capture( + admitted.candidate, + &admitted.original, + )?, }, }; let witness = admitted.witness.take(); @@ -62,6 +66,10 @@ impl<'a> Context<'a> { _ => return Err(refused()), } let committed = self.committed.borrow(); + committed + .intent + .environment + .verify(admitted.candidate, &admitted.original)?; let path = admitted.root.join(FILE); match &committed.witness { Some((identity, bytes)) @@ -219,12 +227,14 @@ pub struct Outcome { } pub fn recover(candidate: &Candidate, options: Options<'_>) -> Result { recover_using(candidate, options, |context, environment_retired| { + let inventory = context.committed.borrow().intent.environment.clone(); runtime::cleanup_recovery( candidate, context.admitted.run, &context.admitted.original, &|| context.guard(), environment_retired, + &inventory, &|snapshot| context.finish(snapshot), ) }) diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs index ec326041b..9d67cca9a 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs @@ -159,6 +159,11 @@ impl Fixture { progress: Progress::Cleanup, receipt_progress: 0, resource_progress: resource_progress(&receipt).unwrap(), + environment: crate::provider::native_environment::Inventory::capture( + &self.candidate, + &receipt, + ) + .unwrap(), } } fn store(&self, intent: &Intent) { @@ -698,6 +703,76 @@ fn recovery_commits_original_before_driver_and_archives_only_after_removed_absen assert!(!fixture.candidate.state_root.join("run/smolvm").exists()); } +#[test] +fn committed_environment_inventory_refuses_record_loss_or_replacement_before_archives() { + for case in ["deleted", "same-bytes-new-inode", "changed-metadata"] { + let fixture = Fixture::new(); + fixture.dead(); + let receipt = fixture.receipt(); + let slot = format!("hack-env-lease-{}-{}", receipt.boot, "a".repeat(32)); + let path = fixture + .candidate + .state_root + .join("run/native-environment-leases") + .join(format!("{slot}.json")); + state::private_directory(path.parent().unwrap()).unwrap(); + let metadata = json!({"version":2,"kind":"native-environment-allocation","binding":{ + "namespace":receipt.review.scope().namespace,"run":RUN,"review":receipt.review.review_id(),"container":receipt.resources["container:web"].name + },"service":"web","uid":0,"gid":0,"slot":slot,"incarnation":receipt.owner,"boot":receipt.boot}); + state::write(&path, &metadata).unwrap(); + let selected = select(&fixture.candidate, RUN).unwrap(); + assert!( + cleanup::recover_using( + &fixture.candidate, + recovery_options(&selected), + |context, _| { + let original = native_input::read_file(&path, 4096)?; + let identity = id(&path)?; + context.guard()?; + let receipt = removed(&fixture); + context.guard()?; + match case { + "deleted" => fs::remove_file(&path).unwrap(), + "same-bytes-new-inode" => { + fs::rename(&path, fixture.root.join("retained-native-allocation")) + .unwrap(); + let mut replacement = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path) + .unwrap(); + replacement.write_all(&original).unwrap(); + replacement.sync_all().unwrap(); + assert_ne!(id(&path).unwrap(), identity); + } + "changed-metadata" => { + let mut changed = metadata.clone(); + changed["uid"] = json!(10); + fs::write(&path, changed.to_string()).unwrap(); + } + _ => unreachable!(), + } + assert!(context.guard().is_err()); + context.finish_using(&absent_snapshot(receipt.clone()), &|_| { + panic!("changed environment inventory cannot retire publication") + })?; + Ok(receipt) + } + ) + .is_err() + ); + assert!(fixture.owner_root().join("owner.json").exists()); + assert!(fixture.owner_root().join("control.sock").exists()); + let saved: Intent = serde_json::from_slice( + &native_input::read_file(&fixture.journal_root().join(FILE), LIMIT).unwrap(), + ) + .unwrap(); + assert!(saved.progress == Progress::Cleanup); + assert!(select(&fixture.candidate, RUN).is_err()); + } +} + #[test] fn publication_retirement_interruption_retries_only_prior_exact_phase_and_inventory() { for stop in [ diff --git a/packages/runtime-core/src/provider/graph/native/journal.rs b/packages/runtime-core/src/provider/graph/native/journal.rs index 5a9b85572..a04c49d5b 100644 --- a/packages/runtime-core/src/provider/graph/native/journal.rs +++ b/packages/runtime-core/src/provider/graph/native/journal.rs @@ -101,6 +101,10 @@ impl Receipt { pub fn owner(&self) -> &str { &self.owner } + #[cfg(target_os = "macos")] + pub(in crate::provider) fn boot(&self) -> &str { + &self.boot + } pub fn resources(&self) -> &BTreeMap { &self.resources } diff --git a/packages/runtime-core/src/provider/graph/native/runtime.rs b/packages/runtime-core/src/provider/graph/native/runtime.rs index 9476862ed..0a7677afb 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime.rs @@ -658,7 +658,7 @@ pub(super) fn cleanup_guarded( expected: Option<&Receipt>, guard: Option<&dyn Fn() -> Result<(), CandidateError>>, ) -> Result { - cleanup_inner(candidate, run, expected, guard, false, None) + cleanup_inner(candidate, run, expected, guard, false, None, None) } #[cfg(target_os = "macos")] pub(super) fn cleanup_recovery( @@ -667,6 +667,7 @@ pub(super) fn cleanup_recovery( expected: &Receipt, guard: &dyn Fn() -> Result<(), CandidateError>, environment_retired: bool, + inventory: &native_environment::Inventory, finish: &dyn Fn(&Snapshot) -> Result<(), CandidateError>, ) -> Result { cleanup_inner( @@ -675,6 +676,7 @@ pub(super) fn cleanup_recovery( Some(expected), Some(guard), environment_retired, + Some(inventory), Some(finish), ) } @@ -684,6 +686,7 @@ fn cleanup_inner( expected: Option<&Receipt>, guard: Option<&dyn Fn() -> Result<(), CandidateError>>, environment_retired: bool, + inventory: Option<&native_environment::Inventory>, finish: Option<&dyn Fn(&Snapshot) -> Result<(), CandidateError>>, ) -> Result { check_startup(guard)?; @@ -704,6 +707,19 @@ fn cleanup_inner( launcher: None, leases: BTreeMap::new(), }; + #[cfg(target_os = "macos")] + if let Some(inventory) = inventory { + native_environment::verify_inventory( + candidate, + backend.engine.guest(), + &receipt, + inventory, + environment_retired, + guard, + )?; + } + #[cfg(not(target_os = "macos"))] + let _ = inventory; cleanup_using_guarded(&backend, &mut receipt, &root, guard)?; check_startup(guard)?; if !environment_retired { @@ -719,6 +735,17 @@ fn cleanup_inner( &receipt, guard, )?; + #[cfg(target_os = "macos")] + if let Some(inventory) = inventory { + native_environment::verify_inventory( + candidate, + backend.engine.guest(), + &receipt, + inventory, + true, + guard, + )?; + } let guarded = GuardedBackend { backend: &backend, guard, diff --git a/packages/runtime-core/src/provider/native_environment.rs b/packages/runtime-core/src/provider/native_environment.rs index 1f02f8f2b..388d7bb4e 100644 --- a/packages/runtime-core/src/provider/native_environment.rs +++ b/packages/runtime-core/src/provider/native_environment.rs @@ -4,7 +4,11 @@ use super::{ }; use crate::{Candidate, CandidateError}; use serde::{Deserialize, Serialize}; -use std::{fs, path::PathBuf}; +#[cfg(target_os = "macos")] +use sha2::{Digest, Sha256}; +use std::{collections::BTreeMap, fs, path::PathBuf}; +#[cfg(target_os = "macos")] +use std::{collections::BTreeSet, os::unix::fs::MetadataExt}; #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -153,6 +157,195 @@ fn read(candidate: &Candidate, slot: &str) -> Result { validate(&intent, slot)?; Ok(intent) } + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RecordWitness { + identity: (u64, u64), + sha256: String, +} +/// The first current-private slot snapshot becomes immutable retry evidence. +/// Guest enumeration separately detects payloads whose host records were lost. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(in crate::provider) struct Inventory { + directory: Option<(u64, u64)>, + records: BTreeMap, +} +#[cfg(target_os = "macos")] +fn entries(candidate: &Candidate) -> Result, CandidateError> { + if count(candidate)? == 0 { + return Ok(Vec::new()); + } + let mut entries = Vec::new(); + for (index, entry) in fs::read_dir(root(candidate)) + .map_err(|_| refused())? + .enumerate() + { + if index >= 4096 { + return Err(refused()); + } + let path = entry.map_err(|_| refused())?.path(); + let name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(refused)?; + let slot = name.strip_suffix(".json").ok_or_else(refused)?; + let before = fs::symlink_metadata(&path).map_err(|_| refused())?; + let intent = read(candidate, slot)?; + let bytes = native_input::read_file(&path, 4096).map_err(|_| refused())?; + let again: Intent = serde_json::from_slice(&bytes).map_err(|_| refused())?; + validate(&again, slot)?; + let after = fs::symlink_metadata(&path).map_err(|_| refused())?; + if (before.dev(), before.ino()) != (after.dev(), after.ino()) + || serde_json::to_vec(&intent).map_err(|_| refused())? + != serde_json::to_vec(&again).map_err(|_| refused())? + { + return Err(refused()); + } + entries.push(( + intent, + RecordWitness { + identity: (after.dev(), after.ino()), + sha256: format!("{:x}", Sha256::digest(bytes)), + }, + )); + } + Ok(entries) +} +#[cfg(target_os = "macos")] +impl Inventory { + pub(in crate::provider) fn capture( + candidate: &Candidate, + receipt: &super::graph::native::Receipt, + ) -> Result { + let directory = match fs::symlink_metadata(root(candidate)) { + Ok(metadata) => { + state::check_private_directory(&root(candidate)).map_err(|_| refused())?; + Some((metadata.dev(), metadata.ino())) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(_) => return Err(refused()), + }; + let mut records = BTreeMap::new(); + let mut services = BTreeSet::new(); + for (intent, witness) in entries(candidate)? { + if intent.binding.run != receipt.review().scope().run { + continue; + } + let resource = receipt + .resources() + .get(&format!("container:{}", intent.service)) + .ok_or_else(refused)?; + if records.len() == super::environment::MAX_MANAGED_SERVICES + || !services.insert(intent.service.clone()) + || intent.incarnation != receipt.owner() + || intent.boot != receipt.boot() + || intent.binding.namespace != receipt.review().scope().namespace + || intent.binding.review != receipt.review().review_id() + || intent.binding.container != resource.name + { + return Err(refused()); + } + records.insert(intent.slot, witness); + } + Ok(Self { directory, records }) + } + pub(in crate::provider) fn verify( + &self, + candidate: &Candidate, + receipt: &super::graph::native::Receipt, + ) -> Result<(), CandidateError> { + if &Self::capture(candidate, receipt)? != self { + return Err(refused()); + } + Ok(()) + } +} + +#[cfg(target_os = "macos")] +fn known_slots( + candidate: &Candidate, + incarnation: &str, + boot: &str, +) -> Result, CandidateError> { + let mut known = environment_recovery::known_current_slots(candidate, incarnation, boot)?; + for (intent, _) in entries(candidate)? { + if intent.incarnation == incarnation && intent.boot == boot && !known.insert(intent.slot) { + return Err(refused()); + } + } + Ok(known) +} +#[cfg(target_os = "macos")] +fn admit_guest_slots( + output: &str, + boot: &str, + known: &BTreeSet, + absent: Option<&Inventory>, +) -> Result<(), CandidateError> { + let body = output + .strip_prefix("environment-slot-inventory-v1\n") + .ok_or_else(refused)?; + if !body.is_empty() && !body.ends_with('\n') { + return Err(refused()); + } + let mut observed = BTreeSet::new(); + for slot in body.lines() { + if observed.len() == super::environment::MAX_CONCURRENT_ALLOCATIONS + || !environment_recovery::valid_slot(slot) + || !slot.starts_with(&format!("hack-env-lease-{boot}-")) + || !known.contains(slot) + || !observed.insert(slot) + || absent.is_some_and(|inventory| inventory.records.contains_key(slot)) + { + return Err(refused()); + } + } + Ok(()) +} +/// Unknown guest slots refuse even when host records are empty. Names only are +/// observed; the retained guest lease independently verifies incarnation/boot. +#[cfg(target_os = "macos")] +pub(in crate::provider) fn verify_inventory( + candidate: &Candidate, + guest: &OwnedGuest<'_>, + receipt: &super::graph::native::Receipt, + inventory: &Inventory, + absent: bool, + guard: Option<&dyn Fn() -> Result<(), CandidateError>>, +) -> Result<(), CandidateError> { + check_guard(guard)?; + inventory.verify(candidate, receipt)?; + let known = known_slots(candidate, guest.incarnation(), guest.boot_id())?; + check_guard(guard)?; + let result = guest.execute_cleanup( + LIST_SLOTS, + &[&super::environment::MAX_CONCURRENT_ALLOCATIONS.to_string()], + ); + check_guard(guard)?; + inventory.verify(candidate, receipt)?; + admit_guest_slots( + &result?, + guest.boot_id(), + &known, + absent.then_some(inventory), + ) +} +#[cfg(target_os = "macos")] +const LIST_SLOTS: &str = r#" +count=0 +printf 'environment-slot-inventory-v1\n' +for path in /run/hack-env-lease-*; do + if test -e "$path" || test -L "$path"; then + test ! -L "$path" + test -d "$path" + count=$((count + 1)) + test "$count" -le "$1" + printf '%s\n' "${path#/run/}" + fi +done +"#; pub(super) fn retire( candidate: &Candidate, guest: &OwnedGuest<'_>, diff --git a/packages/runtime-core/src/provider/native_environment/tests.rs b/packages/runtime-core/src/provider/native_environment/tests.rs index e362355ec..928629fa9 100644 --- a/packages/runtime-core/src/provider/native_environment/tests.rs +++ b/packages/runtime-core/src/provider/native_environment/tests.rs @@ -89,3 +89,95 @@ fn native_and_compose_history_share_capacity_even_when_compose_history_is_absent assert!(environment_recovery::preflight_records(&fixture.candidate, 1).is_ok()); assert!(environment_recovery::preflight_records(&fixture.candidate, 2).is_err()); } + +#[cfg(target_os = "macos")] +#[test] +fn guest_inventory_requires_known_slots_and_exact_selected_absence() { + let fixture = Fixture::new(); + let native = intent(); + let slot = native["slot"].as_str().unwrap(); + let boot = native["boot"].as_str().unwrap(); + let incarnation = native["incarnation"].as_str().unwrap(); + let path = root(&fixture.candidate).join(format!("{slot}.json")); + state::private_directory(path.parent().unwrap()).unwrap(); + state::write(&path, &native).unwrap(); + let compose_slot = format!("hack-env-lease-{boot}-{}", "f".repeat(32)); + let compose_path = fixture + .candidate + .state_root + .join("run/environment-leases") + .join(format!("{compose_slot}.json")); + state::private_directory(compose_path.parent().unwrap()).unwrap(); + state::write(&compose_path, &json!({"version":1,"service":"sibling","slot":compose_slot,"incarnation":incarnation,"boot":boot})).unwrap(); + let known = known_slots(&fixture.candidate, incarnation, boot).unwrap(); + let output = format!("environment-slot-inventory-v1\n{slot}\n{compose_slot}\n"); + assert!(admit_guest_slots(&output, boot, &known, None).is_ok()); + let selected = Inventory { + directory: None, + records: BTreeMap::from([( + slot.into(), + RecordWitness { + identity: (1, 2), + sha256: "a".repeat(64), + }, + )]), + }; + assert!(admit_guest_slots(&output, boot, &known, Some(&selected)).is_err()); + assert!( + admit_guest_slots( + &format!("environment-slot-inventory-v1\n{compose_slot}\n"), + boot, + &known, + Some(&selected) + ) + .is_ok() + ); + fs::remove_file(&path).unwrap(); + let missing = known_slots(&fixture.candidate, incarnation, boot).unwrap(); + assert!(admit_guest_slots(&output, boot, &missing, None).is_err()); + fs::remove_file(&compose_path).unwrap(); + assert!( + known_slots(&fixture.candidate, incarnation, boot) + .unwrap() + .is_empty() + ); + assert!(admit_guest_slots(&output, boot, &BTreeSet::new(), None).is_err()); + for output in [ + format!("environment-slot-inventory-v1\n{compose_slot}\n{compose_slot}\n"), + format!("environment-slot-inventory-v1\n{compose_slot}"), + "environment-slot-inventory-v1\nunknown-private-canary\n".into(), + ] { + assert!(admit_guest_slots(&output, boot, &known, None).is_err()); + } + let many: BTreeSet = (0..=super::super::environment::MAX_CONCURRENT_ALLOCATIONS) + .map(|index| format!("hack-env-lease-{boot}-{index:032x}")) + .collect(); + assert!( + admit_guest_slots( + &format!( + "environment-slot-inventory-v1\n{}\n", + many.iter() + .map(String::as_str) + .collect::>() + .join("\n") + ), + boot, + &many, + None + ) + .is_err() + ); + assert!( + admit_guest_slots( + &format!("environment-slot-inventory-v1\n{compose_slot}\n"), + "22345678-abcd-abcd-abcd-123456789abc", + &known, + None + ) + .is_err() + ); + // A retained pending record cannot become completeness evidence or be promoted. + state::write(&compose_path.with_extension("pending"), &json!({"version":1,"service":"sibling","slot":compose_slot,"incarnation":incarnation,"boot":boot})).unwrap(); + assert!(known_slots(&fixture.candidate, incarnation, boot).is_err()); + assert!(!compose_path.exists()); +} From 397dcc81e4372f3bb662f78906f849e5f7779677 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 14:38:57 -0400 Subject: [PATCH 14/47] feat(runtime): expose exact native owner recovery command --- packages/runtime-core/README.md | 6 + packages/runtime-core/src/graph_cli/native.rs | 40 ++++- .../tests/native_graph_cli_contract.rs | 142 ++++++++++++++++++ 3 files changed, 186 insertions(+), 2 deletions(-) diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index cbb4acecb..3570fd6cf 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -151,6 +151,12 @@ hashes and inventory remain fixed while cleanup phases advance. Completed retry checks absence rather than repeating retirement. Pending writes, unexpected replacements or missing original and archive paths retain a refusal. Public frontend recovery and actual dead-owner acceptance remain separate gates. +The distinct macOS command `graph native recover-live-owner --run-id ID +--expect-receipt SHA --expect-owner SHA --json` exposes only this explicit +cleanup entrypoint. Both hashes select the original read-only recovery envelope; +source, private stdin, review, action and timeout options refuse. Existing live +control and direct-cleanup ownership gates remain strict. This command does not +retire frontend start/ready/source artifacts or enable ordinary `up` takeover. Guarded native cleanup checks its retained authority before and after every engine observation, removal request and admitted stop batch, including failed diff --git a/packages/runtime-core/src/graph_cli/native.rs b/packages/runtime-core/src/graph_cli/native.rs index 2bebc4365..025f9a3ed 100644 --- a/packages/runtime-core/src/graph_cli/native.rs +++ b/packages/runtime-core/src/graph_cli/native.rs @@ -9,7 +9,7 @@ use std::{ fn refused() -> CandidateError { CandidateError::new( "native_graph_arguments", - "Use graph native plan --source-file FILE, run|serve --source-file FILE --expect-review SHA [--environment-stdin] [--timeout-seconds 1..300], inspect|cleanup|recovery-selection --run-id ID, or control --run-id ID --action status|cleanup; optional --json. Foreground serve/control and read-only recovery selection require macOS. Native source and private envelopes require their exact kind/version; values omitted.", + "Use graph native plan --source-file FILE, run|serve --source-file FILE --expect-review SHA [--environment-stdin] [--timeout-seconds 1..300], inspect|cleanup|recovery-selection --run-id ID, recover-live-owner --run-id ID --expect-receipt SHA --expect-owner SHA, or control --run-id ID --action status|cleanup; optional --json. Foreground ownership and its recovery require macOS. Native source and private envelopes require their exact kind/version; values omitted.", ) } fn hex(value: &str, len: usize) -> bool { @@ -35,7 +35,7 @@ pub(super) fn command(candidate: &Candidate, args: &[&str]) -> Result { + | "--action" | "--expect-receipt" | "--expect-owner") => { let value = *args .get(index + 1) .filter(|value| !value.starts_with("--")) @@ -48,6 +48,40 @@ pub(super) fn command(candidate: &Candidate, args: &[&str]) -> Result return Err(refused()), } } + if *action == "recover-live-owner" { + if private || singles.len() != 3 { + return Err(refused()); + } + let run = *singles + .get("--run-id") + .filter(|run| hex(run, 32)) + .ok_or_else(refused)?; + let receipt = *singles + .get("--expect-receipt") + .filter(|receipt| hex(receipt, 64)) + .ok_or_else(refused)?; + let owner = *singles + .get("--expect-owner") + .filter(|owner| hex(owner, 64)) + .ok_or_else(refused)?; + #[cfg(target_os = "macos")] + { + return serde_json::to_value(native::foreground::recovery::recover( + candidate, + native::foreground::recovery::Options { + run, + expect_receipt: receipt, + expect_owner: owner, + }, + )?) + .map_err(|_| refused()); + } + #[cfg(not(target_os = "macos"))] + { + let _ = (run, receipt, owner); + return Err(foreground_unavailable()); + } + } if *action == "recovery-selection" { if private || singles.len() != 1 { return Err(refused()); @@ -118,6 +152,8 @@ pub(super) fn command(candidate: &Candidate, args: &[&str]) -> Result Output { + let mut child = self + .command(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let unread = child.stdin.take().unwrap(); + let output = child.wait_with_output().unwrap(); + drop(unread); + output + } fn source_path(&self) -> &str { self.source.to_str().unwrap() } @@ -134,6 +147,135 @@ impl Drop for Fixture { } } +#[test] +fn public_native_recovery_requires_exact_closed_selectors_and_refuses_absent_authority() { + let fixture = Fixture::new(); + let run = "b".repeat(32); + let receipt = "c".repeat(64); + let owner = "d".repeat(64); + let selected = [ + "graph", + "native", + "recover-live-owner", + "--run-id", + &run, + "--expect-receipt", + &receipt, + "--expect-owner", + &owner, + "--json", + ]; + let output = fixture.invoke(&selected); + assert!(!output.status.success()); + #[cfg(target_os = "macos")] + assert!(String::from_utf8_lossy(&output.stderr).contains("native_graph_live_owner_recovery")); + #[cfg(not(target_os = "macos"))] + assert!( + String::from_utf8_lossy(&output.stderr).contains("native_graph_foreground_unsupported") + ); + fixture.assert_no_state(); + for extra in [ + vec!["--environment-stdin"], + vec!["--json"], + vec!["--run-id", &run], + vec!["--source-file", fixture.source_path()], + vec!["--expect-review", &receipt], + vec!["--timeout-seconds", "1"], + vec!["--action", "cleanup"], + ] { + let mut args = selected.to_vec(); + args.extend(extra); + // A held empty pipe distinguishes argument refusal from private delivery. + let output = fixture.invoke_before_stdin(&args); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("native_graph_arguments")); + fixture.assert_no_state(); + } + for bad in [ + vec!["graph", "native", "recover-live-owner", "--run-id", &run], + vec![ + "graph", + "native", + "recover-live-owner", + "--run-id", + &run, + "--expect-receipt", + &receipt, + ], + vec![ + "graph", + "native", + "recover-live-owner", + "--run-id", + &run, + "--expect-receipt", + "private-recovery-cli-canary", + "--expect-owner", + &owner, + ], + vec![ + "graph", + "native", + "recover-live-owner", + "--run-id", + "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", + "--expect-receipt", + &receipt, + "--expect-owner", + &owner, + ], + ] { + let output = fixture.invoke(&bad); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("native_graph_arguments")); + assert!(!String::from_utf8_lossy(&output.stderr).contains("private-recovery-cli-canary")); + assert!(!String::from_utf8_lossy(&output.stdout).contains("private-recovery-cli-canary")); + fixture.assert_no_state(); + } +} + +#[test] +fn recovery_hash_flags_never_widen_existing_native_actions() { + let fixture = Fixture::new(); + let hash = "c".repeat(64); + for action in ["plan", "run", "serve"] { + for flag in ["--expect-receipt", "--expect-owner"] { + let output = fixture.invoke(&[ + "graph", + "native", + action, + "--source-file", + fixture.source_path(), + flag, + &hash, + "--json", + ]); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("native_graph_arguments")); + fixture.assert_no_state(); + } + } + for action in ["inspect", "cleanup", "recovery-selection", "control"] { + let mut args = vec![ + "graph", + "native", + action, + "--run-id", + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "--expect-owner", + &hash, + "--json", + ]; + if action == "control" { + args.extend(["--action", "status"]); + } + let output = fixture.invoke(&args); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("native_graph_arguments")); + fixture.assert_no_state(); + } +} + #[test] fn public_native_file_intent_refuses_before_private_stdin_or_candidate_state() { let fixture = Fixture::new(); From 8a55df2ddc58fb994bbd36ae6281593528c3440b Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 14:41:33 -0400 Subject: [PATCH 15/47] test(runtime): bound native recovery argument refusal checks --- .../runtime-core/src/provider/graph/native/runtime.rs | 3 ++- .../runtime-core/tests/native_graph_cli_contract.rs | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/runtime-core/src/provider/graph/native/runtime.rs b/packages/runtime-core/src/provider/graph/native/runtime.rs index 0a7677afb..f65f8b4d8 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime.rs @@ -680,6 +680,7 @@ pub(super) fn cleanup_recovery( Some(finish), ) } +type CleanupFinish<'a> = dyn Fn(&Snapshot) -> Result<(), CandidateError> + 'a; fn cleanup_inner( candidate: &Candidate, run: &str, @@ -687,7 +688,7 @@ fn cleanup_inner( guard: Option<&dyn Fn() -> Result<(), CandidateError>>, environment_retired: bool, inventory: Option<&native_environment::Inventory>, - finish: Option<&dyn Fn(&Snapshot) -> Result<(), CandidateError>>, + finish: Option<&CleanupFinish<'_>>, ) -> Result { check_startup(guard)?; let engine = Engine::connect_cleanup(candidate)?; diff --git a/packages/runtime-core/tests/native_graph_cli_contract.rs b/packages/runtime-core/tests/native_graph_cli_contract.rs index 0ac5258ac..8e27f672a 100644 --- a/packages/runtime-core/tests/native_graph_cli_contract.rs +++ b/packages/runtime-core/tests/native_graph_cli_contract.rs @@ -8,6 +8,7 @@ use std::{ path::PathBuf, process::{Command, Output, Stdio}, sync::atomic::{AtomicU64, Ordering}, + time::{Duration, Instant}, }; struct Fixture { root: PathBuf, @@ -84,6 +85,15 @@ impl Fixture { .spawn() .unwrap(); let unread = child.stdin.take().unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while child.try_wait().unwrap().is_none() { + if Instant::now() >= deadline { + child.kill().unwrap(); + child.wait().unwrap(); + panic!("argument refusal failed to settle before the owned fixture deadline"); + } + std::thread::sleep(Duration::from_millis(5)); + } let output = child.wait_with_output().unwrap(); drop(unread); output From d29c97adcf306cda3ba1121854df8e182a2b83ca Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 14:47:41 -0400 Subject: [PATCH 16/47] feat(cli): decode closed native recovery selections --- .../native-authored-recovery-protocol.ts | 147 ++++++++++++ .../native-authored-recovery-protocol.test.ts | 217 ++++++++++++++++++ 2 files changed, 364 insertions(+) create mode 100644 src/backends/native-authored-recovery-protocol.ts create mode 100644 tests/native-authored-recovery-protocol.test.ts diff --git a/src/backends/native-authored-recovery-protocol.ts b/src/backends/native-authored-recovery-protocol.ts new file mode 100644 index 000000000..f0effc50a --- /dev/null +++ b/src/backends/native-authored-recovery-protocol.ts @@ -0,0 +1,147 @@ +import { isRecord } from "../lib/guards.ts"; +import { + type NativeAuthoredReceipt, + nativeAuthoredReceiptBinding, + parseNativeAuthoredReceipt, +} from "./native-authored-graph-protocol.ts"; + +const HEX64 = /^[a-f0-9]{64}$/; +export type NativeAuthoredRecoverySelection = { + readonly version: 1; + readonly kind: "native-graph-recovery-selection"; + readonly run: string; + readonly receipt: NativeAuthoredReceipt; + readonly receipt_sha256: string; + readonly owner_sha256: string; + readonly host_boot_micros: number; +}; +export type NativeAuthoredRecoveryResult = { + readonly version: 1; + readonly kind: "native-graph-live-owner-recovered"; + readonly run: string; + readonly same_boot: true; + readonly publication_retired: true; + readonly receipt: NativeAuthoredReceipt; +}; +function refused(): never { + throw new Error( + "Native recovery response is invalid or changed; values omitted." + ); +} +function fields( + value: unknown, + names: readonly string[] +): value is Record { + return ( + isRecord(value) && + names.every((name) => Object.hasOwn(value, name)) && + Object.keys(value).length === names.length + ); +} +function hash(value: unknown): value is string { + return typeof value === "string" && HEX64.test(value); +} +function ready(value: unknown): NativeAuthoredReceipt { + const receipt = parseNativeAuthoredReceipt(value); + if ( + receipt.phase !== "ready-observed" || + receipt.failure !== undefined || + Object.values(receipt.resources).some((resource) => resource.id === null) + ) { + return refused(); + } + return receipt; +} + +/** Raw selectors belong to Rust's secure reader. This copied, closed envelope + * binds the original durable frontend Ready membership; it grants no effects. */ +export function parseNativeAuthoredRecoverySelection(opts: { + readonly value: unknown; + readonly admitted: NativeAuthoredReceipt; +}): NativeAuthoredRecoverySelection { + const admitted = ready(opts.admitted); + const value = opts.value; + if ( + !fields(value, [ + "version", + "kind", + "run", + "receipt", + "receipt_sha256", + "owner_sha256", + "host_boot_micros", + ]) || + value.version !== 1 || + value.kind !== "native-graph-recovery-selection" || + value.run !== admitted.review.provenance.run || + !hash(value.receipt_sha256) || + !hash(value.owner_sha256) || + typeof value.host_boot_micros !== "number" || + !Number.isSafeInteger(value.host_boot_micros) || + value.host_boot_micros <= 0 + ) { + return refused(); + } + const receipt = ready(value.receipt); + if ( + nativeAuthoredReceiptBinding(receipt) !== + nativeAuthoredReceiptBinding(admitted) + ) { + return refused(); + } + return { + version: 1, + kind: "native-graph-recovery-selection", + run: value.run, + receipt, + receipt_sha256: value.receipt_sha256, + owner_sha256: value.owner_sha256, + host_boot_micros: value.host_boot_micros, + }; +} + +/** Matching Removed is a cleanup result, not frontend retirement authority. + * The caller must also inspect current null observations and retain its leases. */ +export function parseNativeAuthoredRecoveryResult(opts: { + readonly value: unknown; + readonly expected: NativeAuthoredRecoverySelection; +}): NativeAuthoredRecoveryResult { + const expected = parseNativeAuthoredRecoverySelection({ + value: opts.expected, + admitted: opts.expected.receipt, + }); + const value = opts.value; + if ( + !fields(value, [ + "version", + "kind", + "run", + "same_boot", + "publication_retired", + "receipt", + ]) || + value.version !== 1 || + value.kind !== "native-graph-live-owner-recovered" || + value.run !== expected.run || + value.same_boot !== true || + value.publication_retired !== true + ) { + return refused(); + } + const receipt = parseNativeAuthoredReceipt(value.receipt); + if ( + receipt.phase !== "removed" || + nativeAuthoredReceiptBinding(receipt) !== + nativeAuthoredReceiptBinding(expected.receipt) + ) { + return refused(); + } + return { + version: 1, + kind: "native-graph-live-owner-recovered", + run: expected.run, + same_boot: true, + publication_retired: true, + receipt, + }; +} diff --git a/tests/native-authored-recovery-protocol.test.ts b/tests/native-authored-recovery-protocol.test.ts new file mode 100644 index 000000000..21f0c5937 --- /dev/null +++ b/tests/native-authored-recovery-protocol.test.ts @@ -0,0 +1,217 @@ +import { expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { parseNativeAuthoredReceipt } from "../src/backends/native-authored-graph-protocol.ts"; +import { + parseNativeAuthoredRecoveryResult, + parseNativeAuthoredRecoverySelection, +} from "../src/backends/native-authored-recovery-protocol.ts"; + +function receipt() { + const run = "a".repeat(32); + const provenance = { + version: 1, + kind: "native", + namespace: "b".repeat(64), + run, + input: { + semantic_hash: "c".repeat(64), + local_resolution_hash: "d".repeat(64), + environment_policy_hash: "e".repeat(64), + selected_profiles: [], + }, + }; + return { + version: 2, + kind: "native-graph-runtime", + owner: "f".repeat(32), + boot: "00000000-0000-0000-0000-000000000001", + review: { + provenance, + review_id: createHash("sha256") + .update("hack.native-graph-review/v1\0") + .update(JSON.stringify(provenance)) + .digest("hex"), + }, + phase: "ready-observed", + readiness: { web: "healthy" }, + resources: { + "network:default": { + kind: "network", + key: "default", + name: `hkn-${run}-network-0`, + id: "1".repeat(64), + image: null, + phase: "created", + outbound: true, + }, + "container:web": { + kind: "container", + key: "web", + name: `hkn-${run}-container-0`, + id: "2".repeat(64), + image: `sha256:${"3".repeat(64)}`, + phase: "started", + networks: ["default"], + }, + }, + }; +} +function selection() { + const ready = receipt(); + return { + version: 1, + kind: "native-graph-recovery-selection", + run: ready.review.provenance.run, + receipt: ready, + receipt_sha256: "4".repeat(64), + owner_sha256: "5".repeat(64), + host_boot_micros: 1_791_000_000_000_000, + }; +} +function result() { + const removed = receipt(); + removed.phase = "removed"; + for (const resource of Object.values(removed.resources)) { + resource.phase = "removed"; + } + return { + version: 1, + kind: "native-graph-live-owner-recovered", + run: removed.review.provenance.run, + same_boot: true, + publication_retired: true, + receipt: removed, + }; +} +test("native recovery copies original selectors and binds exact Removed to durable Ready", () => { + const input = selection(); + const admitted = parseNativeAuthoredReceipt(receipt()); + const expected = parseNativeAuthoredRecoverySelection({ + value: input, + admitted, + }); + input.receipt.resources["container:web"].id = "9".repeat(64); + input.receipt_sha256 = "a".repeat(64); + expect(expected.receipt.resources["container:web"]?.id).toBe("2".repeat(64)); + expect(expected.receipt_sha256).toBe("4".repeat(64)); + expect( + parseNativeAuthoredRecoveryResult({ value: result(), expected }).receipt + .phase + ).toBe("removed"); +}); +test("native recovery selection refuses inherited unknown private or invalid boot/hash fields", () => { + const admitted = parseNativeAuthoredReceipt(receipt()); + const canary = "private-recovery-wire-canary"; + for (const value of [ + { ...selection(), values: canary }, + { ...selection(), owner: canary }, + { ...selection(), version: 2 }, + { ...selection(), kind: "compose-graph-recovery-selection" }, + { ...selection(), host_boot_micros: 0 }, + { ...selection(), host_boot_micros: Number.MAX_SAFE_INTEGER + 1 }, + { ...selection(), host_boot_micros: 1.5 }, + { ...selection(), host_boot_micros: "1" }, + { ...selection(), receipt_sha256: canary }, + { ...selection(), owner_sha256: "F".repeat(64) }, + Object.create(selection()), + ]) { + try { + parseNativeAuthoredRecoverySelection({ value, admitted }); + throw new Error("fixture admitted forbidden selection"); + } catch (error) { + expect(String(error)).toContain("invalid"); + expect(String(error)).not.toContain(canary); + } + } +}); +test("native recovery rejects self-consistent foreign membership and incomplete original Ready", () => { + const admitted = parseNativeAuthoredReceipt(receipt()); + for (const change of [ + "id", + "image", + "owner", + "boot", + "readiness", + "network", + ]) { + const selected = selection(); + const changed = selected.receipt; + if (change === "id") { + changed.resources["container:web"].id = "9".repeat(64); + } + if (change === "image") { + changed.resources["container:web"].image = `sha256:${"9".repeat(64)}`; + } + if (change === "owner") { + changed.owner = "9".repeat(32); + } + if (change === "boot") { + changed.boot = "00000000-0000-0000-0000-000000000002"; + } + if (change === "readiness") { + changed.readiness.web = "started"; + } + if (change === "network") { + changed.resources["network:default"].outbound = false; + } + expect(() => + parseNativeAuthoredRecoverySelection({ value: selected, admitted }) + ).toThrow("invalid"); + // The same attack in a valid Removed reply still fails original membership. + changed.phase = "removed"; + for (const resource of Object.values(changed.resources)) { + resource.phase = "removed"; + } + const expected = parseNativeAuthoredRecoverySelection({ + value: selection(), + admitted, + }); + expect(() => + parseNativeAuthoredRecoveryResult({ + value: { ...result(), receipt: changed }, + expected, + }) + ).toThrow("invalid"); + } + for (const phase of ["preparing", "stop-intent", "removed"]) { + const selected = selection(); + selected.receipt.phase = phase; + expect(() => + parseNativeAuthoredRecoverySelection({ value: selected, admitted }) + ).toThrow("invalid"); + } +}); +test("native recovered result closes outcome flags unknown fields and nested private diagnostics", () => { + const expected = parseNativeAuthoredRecoverySelection({ + value: selection(), + admitted: parseNativeAuthoredReceipt(receipt()), + }); + const canary = "private-recovery-result-canary"; + for (const value of [ + { ...result(), publication_retired: false }, + { ...result(), same_boot: false }, + { ...result(), run: "9".repeat(32) }, + { ...result(), values: canary }, + { ...result(), receipt: receipt() }, + Object.create(result()), + { + ...result(), + receipt: { + ...result().receipt, + failure: { + service: "web", + observation: { state: "exited", code: 1, values: canary }, + }, + }, + }, + ]) { + expect(() => + parseNativeAuthoredRecoveryResult({ value, expected }) + ).toThrow("invalid"); + try { + parseNativeAuthoredRecoveryResult({ value, expected }); + } catch (error) { + expect(String(error)).not.toContain(canary); + } + } +}); From 7400c21aab0a2b3f3204d205134e8c205047df4b Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 15:03:49 -0400 Subject: [PATCH 17/47] feat: bind explicit frontend lock recovery to private selection --- src/lib/native-compose-private-state.ts | 240 ++++++++++++++++----- tests/native-private-lock-recovery.test.ts | 185 ++++++++++++++++ 2 files changed, 373 insertions(+), 52 deletions(-) create mode 100644 tests/native-private-lock-recovery.test.ts diff --git a/src/lib/native-compose-private-state.ts b/src/lib/native-compose-private-state.ts index a773b9abd..264f67881 100644 --- a/src/lib/native-compose-private-state.ts +++ b/src/lib/native-compose-private-state.ts @@ -1,4 +1,4 @@ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { constants, type Stats } from "node:fs"; import { type FileHandle, @@ -14,6 +14,7 @@ import { join } from "node:path"; import { isRecord } from "./guards.ts"; const TOKEN = /^[a-f0-9]{32}$/; +const SHA256 = /^[a-f0-9]{64}$/; const BOOT_ID = /^[a-f0-9]{8}-(?:[a-f0-9]{4}-){3}[a-f0-9]{12}$/; const BIRTH = /^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun) (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{1,2} \d{2}:\d{2}:\d{2} \d{4}$/; @@ -41,6 +42,93 @@ type LockOwner = { readonly bootId: string; readonly birth: string; }; +/** Private current snapshot, independently re-admitted before explicit unlink. + * This is neither a historical inode witness nor permission to take a live lock. */ +export type NativeComposeInterruptedLockSelection = { + readonly version: 1; + readonly kind: "native-private-interrupted-lock"; + readonly directory: { readonly dev: number; readonly ino: number }; + readonly file: { + readonly dev: number; + readonly ino: number; + readonly sha256: string; + }; + readonly owner: LockOwner; +}; + +function fileIdentity(value: unknown): value is Record & { + readonly dev: number; + readonly ino: number; +} { + return ( + isRecord(value) && + typeof value.dev === "number" && + Number.isSafeInteger(value.dev) && + value.dev >= 0 && + typeof value.ino === "number" && + Number.isSafeInteger(value.ino) && + value.ino > 0 + ); +} +export function parseNativeComposeInterruptedLockSelection( + value: unknown +): NativeComposeInterruptedLockSelection { + if ( + !(isRecord(value) && keys(value, "directory,file,kind,owner,version")) || + value.version !== 1 || + value.kind !== "native-private-interrupted-lock" || + !fileIdentity(value.directory) || + !keys(value.directory, "dev,ino") || + !fileIdentity(value.file) || + !keys(value.file, "dev,ino,sha256") || + typeof value.file.sha256 !== "string" || + !SHA256.test(value.file.sha256) + ) { + return refuse(); + } + const owner = parseLockOwner(JSON.stringify(value.owner)); + return Object.freeze({ + version: 1, + kind: "native-private-interrupted-lock", + directory: Object.freeze({ + dev: value.directory.dev, + ino: value.directory.ino, + }), + file: Object.freeze({ + dev: value.file.dev, + ino: value.file.ino, + sha256: value.file.sha256, + }), + owner: Object.freeze(owner), + }); +} +function interruptedSelection( + lock: HeldDirectory, + original: { readonly info: Stats; readonly text: string } +): NativeComposeInterruptedLockSelection { + return parseNativeComposeInterruptedLockSelection({ + version: 1, + kind: "native-private-interrupted-lock", + directory: { dev: lock.info.dev, ino: lock.info.ino }, + file: { + dev: original.info.dev, + ino: original.info.ino, + sha256: createHash("sha256").update(original.text).digest("hex"), + }, + owner: parseLockOwner(original.text), + }); +} +function requireInterruptedSelection( + expected: NativeComposeInterruptedLockSelection | undefined, + current: NativeComposeInterruptedLockSelection +): void { + if ( + expected !== undefined && + JSON.stringify(current) !== JSON.stringify(expected) + ) { + refuse(); + } +} async function inspection( command: readonly string[] @@ -518,66 +606,114 @@ export function createNativeComposePrivateMutationLock(opts: { } }; - return { - withLock, - /** Native startup admission uses the same active lease without material authority. */ - withHeldLock: (run: (assertHeld: () => Promise) => Promise) => - withLock((lease) => run(lease.assertHeld)), - async recoverInterruptedLock() { + const selectInterruptedLock = async () => { + await check(); + await requireAbsentGuard(recoveryPath); + const lock = await holdDirectory(lockPath, true); + try { + const ownerPath = join(lockPath, "owner"); + const original = await readPrivate(ownerPath, LOCK_OWNER_LIMIT); + const selected = interruptedSelection(lock, original); + await requireDeadOwner(selected.owner); await check(); await requireAbsentGuard(recoveryPath); - try { - await lstat(lockPath); - } catch (error) { - if (hasCode(error, "ENOENT")) { - return; - } - throw error; + await recheckDirectories([lock]); + const latest = await readPrivate(ownerPath, LOCK_OWNER_LIMIT); + if ( + !sameFile(original.info, latest.info) || + latest.text !== original.text || + JSON.stringify(await readdir(lockPath)) !== '["owner"]' + ) { + return refuse(); } + await requireDeadOwner(selected.owner); + await recheckDirectories([lock]); + return selected; + } finally { + await lock.file.close(); + } + }; + + const recoverInterruptedLock = async ( + expected?: NativeComposeInterruptedLockSelection + ) => { + await check(); + await requireAbsentGuard(recoveryPath); + try { + await lstat(lockPath); + } catch (error) { + if (hasCode(error, "ENOENT") && expected === undefined) { + return; + } + throw error; + } + if (expected !== undefined) { + requireInterruptedSelection(expected, await selectInterruptedLock()); + } + try { + await mkdir(recoveryPath, { mode: 0o700 }); + } catch (error) { + if (hasCode(error, "EEXIST")) { + throw new NativeComposeGenerationError("E_NATIVE_COMPOSE_BUSY"); + } + throw error; + } + const recovery = await holdDirectory(recoveryPath, true); + try { + const lock = await holdDirectory(lockPath, true); try { - await mkdir(recoveryPath, { mode: 0o700 }); - } catch (error) { - if (hasCode(error, "EEXIST")) { - throw new NativeComposeGenerationError("E_NATIVE_COMPOSE_BUSY"); + const ownerPath = join(lockPath, "owner"); + const original = await readPrivate(ownerPath, LOCK_OWNER_LIMIT); + const owner = parseLockOwner(original.text); + if (expected !== undefined) { + requireInterruptedSelection( + expected, + interruptedSelection(lock, original) + ); } - throw error; + await requireDeadOwner(owner); + await check(); + await recheckDirectories([recovery, lock]); + const latest = await readPrivate(ownerPath, LOCK_OWNER_LIMIT); + if ( + !sameFile(original.info, latest.info) || + latest.text !== original.text || + JSON.stringify(await readdir(lockPath)) !== '["owner"]' + ) { + return refuse(); + } + await requireDeadOwner(owner); + await recheckDirectories([recovery, lock]); + await unlink(ownerPath); + await rmdir(lockPath); + await parent?.file.sync(); + } finally { + await lock.file.close(); } - const recovery = await holdDirectory(recoveryPath, true); + } finally { try { - const lock = await holdDirectory(lockPath, true); - try { - const ownerPath = join(lockPath, "owner"); - const original = await readPrivate(ownerPath, LOCK_OWNER_LIMIT); - const owner = parseLockOwner(original.text); - await requireDeadOwner(owner); - await check(); - await recheckDirectories([recovery, lock]); - const latest = await readPrivate(ownerPath, LOCK_OWNER_LIMIT); - if ( - !sameFile(original.info, latest.info) || - latest.text !== original.text || - JSON.stringify(await readdir(lockPath)) !== '["owner"]' - ) { - refuse(); - } - await requireDeadOwner(owner); - await recheckDirectories([recovery, lock]); - await unlink(ownerPath); - await rmdir(lockPath); - await parent?.file.sync(); - } finally { - await lock.file.close(); - } + await check(); + await recheckDirectories([recovery]); + await rmdir(recoveryPath); + await parent?.file.sync(); } finally { - try { - await check(); - await recheckDirectories([recovery]); - await rmdir(recoveryPath); - await parent?.file.sync(); - } finally { - await recovery.file.close(); - } + await recovery.file.close(); } + } + }; + + return { + withLock, + /** Native startup admission uses the same active lease without material authority. */ + withHeldLock: (run: (assertHeld: () => Promise) => Promise) => + withLock((lease) => run(lease.assertHeld)), + recoverInterruptedLock: () => recoverInterruptedLock(), + /** Read-only current dead/same-host-boot snapshot; no lock is created or retired. */ + selectInterruptedLock, + /** Capture caller input before any await; absence or a changed inode never grants retirement. */ + recoverSelectedInterruptedLock(value: unknown) { + const expected = parseNativeComposeInterruptedLockSelection(value); + return recoverInterruptedLock(expected); }, }; } diff --git a/tests/native-private-lock-recovery.test.ts b/tests/native-private-lock-recovery.test.ts new file mode 100644 index 000000000..98a9e3f3f --- /dev/null +++ b/tests/native-private-lock-recovery.test.ts @@ -0,0 +1,185 @@ +import { afterEach, expect, test } from "bun:test"; +import { + chmod, + copyFile, + lstat, + mkdir, + mkdtemp, + readdir, + realpath, + rename, + rm, + unlink, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + createNativeComposePrivateMutationLock, + holdDirectory, + parseNativeComposeInterruptedLockSelection, + recheckDirectories, +} from "../src/lib/native-compose-private-state.ts"; + +const fixtures: Array<{ + readonly root: string; + readonly child: ReturnType; + readonly close: () => Promise; +}> = []; +afterEach(async () => { + for (const fixture of fixtures.splice(0)) { + if (fixture.child.exitCode === null) { + fixture.child.kill("SIGKILL"); + } + await fixture.child.exited; + await fixture.close(); + await rm(fixture.root, { recursive: true, force: true }); + } +}); + +async function fixture() { + const root = await realpath( + await mkdtemp(join(tmpdir(), "native-dead-lock-")) + ); + await chmod(root, 0o700); + const source = join( + import.meta.dir, + "../src/lib/native-compose-private-state.ts" + ); + const program = [ + `import { createNativeComposePrivateMutationLock, holdDirectory, recheckDirectories } from ${JSON.stringify(source)};`, + `const root = ${JSON.stringify(root)};`, + "const parent = await holdDirectory(root, true);", + "const lock = createNativeComposePrivateMutationLock({ lockPath: root+'/held', recoveryPath: root+'/recovering', parent, check: () => recheckDirectories([parent]) });", + "await lock.withLock(async () => {", + "await Bun.write(root+'/ready', 'ready');", + "await new Promise(resolve => setTimeout(resolve, 120000));", + "});", + "await parent.file.close();", + ].join("\n"); + const child = Bun.spawn([process.execPath, "--eval", program], { + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + env: { PATH: process.env.PATH, TMPDIR: process.env.TMPDIR }, + }); + const parent = await holdDirectory(root, true); + fixtures.push({ root, child, close: () => parent.file.close() }); + const deadline = performance.now() + 5000; + while (!(await Bun.file(join(root, "ready")).exists())) { + if (child.exitCode !== null || performance.now() >= deadline) { + throw new Error("Owned lock fixture did not publish its ready witness."); + } + await Bun.sleep(10); + } + const lock = createNativeComposePrivateMutationLock({ + lockPath: join(root, "held"), + recoveryPath: join(root, "recovering"), + parent, + check: () => recheckDirectories([parent]), + }); + return { + root, + child, + lock, + async kill() { + child.kill("SIGKILL"); + expect(await child.exited).toBe(137); + }, + }; +} + +test("selected private lock recovery refuses a live owner and reads a dead owner without effects", async () => { + const current = await fixture(); + await expect(current.lock.selectInterruptedLock()).rejects.toThrow(); + expect(await readdir(join(current.root, "held"))).toEqual(["owner"]); + expect((await readdir(current.root)).includes("recovering")).toBe(false); + await current.kill(); + const selected = await current.lock.selectInterruptedLock(); + expect(selected.owner.pid).toBe(current.child.pid); + expect(selected.file.ino).toBe( + (await lstat(join(current.root, "held/owner"))).ino + ); + expect(await readdir(join(current.root, "held"))).toEqual(["owner"]); + await current.lock.recoverSelectedInterruptedLock(selected); + expect(await readdir(current.root)).toEqual(["ready"]); + await expect( + current.lock.recoverSelectedInterruptedLock(selected) + ).rejects.toThrow(); + // The legacy helper's absent-lock behavior is unchanged. + await current.lock.recoverInterruptedLock(); +}); + +test("selected private lock recovery captures the complete caller selection before its first await", async () => { + const current = await fixture(); + await current.kill(); + const selected = JSON.parse( + JSON.stringify(await current.lock.selectInterruptedLock()) + ); + const pending = current.lock.recoverSelectedInterruptedLock(selected); + selected.file.ino += 1; + selected.owner.bootId = "00000000-0000-0000-0000-000000000000"; + await pending; + expect(await readdir(current.root)).toEqual(["ready"]); +}); + +test("saved private lock selection refuses same-byte replacement owner or directory", async () => { + for (const replacement of ["owner", "directory"]) { + const current = await fixture(); + await current.kill(); + const selected = await current.lock.selectInterruptedLock(); + if (replacement === "owner") { + const path = join(current.root, "held/owner"); + await copyFile(path, join(current.root, "copy")); + await chmod(join(current.root, "copy"), 0o600); + await rename(join(current.root, "copy"), path); + } else { + await rename(join(current.root, "held"), join(current.root, "original")); + await mkdir(join(current.root, "held"), { mode: 0o700 }); + await copyFile( + join(current.root, "original/owner"), + join(current.root, "held/owner") + ); + await chmod(join(current.root, "held/owner"), 0o600); + } + await expect( + current.lock.recoverSelectedInterruptedLock(selected) + ).rejects.toThrow(); + expect(await readdir(join(current.root, "held"))).toEqual(["owner"]); + expect((await readdir(current.root)).includes("recovering")).toBe(false); + } +}); + +test("private dead-lock selectors close nested fields and do not admit another host boot", async () => { + const current = await fixture(); + await current.kill(); + const original = await current.lock.selectInterruptedLock(); + for (const value of [ + { ...original, private: "must-not-reflect" }, + { ...original, file: { ...original.file, path: "must-not-reflect" } }, + { ...original, owner: { ...original.owner, pid: 0 } }, + { ...original, directory: { ...original.directory, ino: -1 } }, + ]) { + expect(() => parseNativeComposeInterruptedLockSelection(value)).toThrow(); + } + const wrongBoot = { + ...original, + owner: { + ...original.owner, + bootId: "00000000-0000-0000-0000-000000000000", + }, + }; + await expect( + current.lock.recoverSelectedInterruptedLock(wrongBoot) + ).rejects.toThrow(); + expect(await readdir(join(current.root, "held"))).toEqual(["owner"]); + await Bun.write( + join(current.root, "held/owner"), + JSON.stringify(wrongBoot.owner) + ); + await expect(current.lock.selectInterruptedLock()).rejects.toThrow(); + expect((await readdir(current.root)).includes("recovering")).toBe(false); + await unlink(join(current.root, "held/owner")); + await expect( + current.lock.recoverSelectedInterruptedLock(original) + ).rejects.toThrow(); +}); From 466646c55fbce7ab724ae59efd731c80e37e1678 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 15:07:03 -0400 Subject: [PATCH 18/47] fix: recheck selected frontend lock before retirement --- src/lib/native-compose-private-state.ts | 78 +++++++++++++--------- tests/native-private-lock-recovery.test.ts | 61 ++++++++++++++++- 2 files changed, 106 insertions(+), 33 deletions(-) diff --git a/src/lib/native-compose-private-state.ts b/src/lib/native-compose-private-state.ts index 264f67881..a59a2377a 100644 --- a/src/lib/native-compose-private-state.ts +++ b/src/lib/native-compose-private-state.ts @@ -634,6 +634,53 @@ export function createNativeComposePrivateMutationLock(opts: { } }; + const retireInterruptedLock = async ( + recovery: HeldDirectory, + expected: NativeComposeInterruptedLockSelection | undefined + ) => { + const lock = await holdDirectory(lockPath, true); + try { + const ownerPath = join(lockPath, "owner"); + const original = await readPrivate(ownerPath, LOCK_OWNER_LIMIT); + const owner = parseLockOwner(original.text); + if (expected !== undefined) { + requireInterruptedSelection( + expected, + interruptedSelection(lock, original) + ); + } + await requireDeadOwner(owner); + await check(); + await recheckDirectories([recovery, lock]); + const latest = await readPrivate(ownerPath, LOCK_OWNER_LIMIT); + if ( + !sameFile(original.info, latest.info) || + latest.text !== original.text || + JSON.stringify(await readdir(lockPath)) !== '["owner"]' + ) { + return refuse(); + } + await requireDeadOwner(owner); + await recheckDirectories([recovery, lock]); + if (expected !== undefined) { + await check(); + await recheckDirectories([recovery, lock]); + const final = await readPrivate(ownerPath, LOCK_OWNER_LIMIT); + if ( + !sameFile(original.info, final.info) || + final.text !== original.text + ) { + return refuse(); + } + } + await unlink(ownerPath); + await rmdir(lockPath); + await parent?.file.sync(); + } finally { + await lock.file.close(); + } + }; + const recoverInterruptedLock = async ( expected?: NativeComposeInterruptedLockSelection ) => { @@ -660,36 +707,7 @@ export function createNativeComposePrivateMutationLock(opts: { } const recovery = await holdDirectory(recoveryPath, true); try { - const lock = await holdDirectory(lockPath, true); - try { - const ownerPath = join(lockPath, "owner"); - const original = await readPrivate(ownerPath, LOCK_OWNER_LIMIT); - const owner = parseLockOwner(original.text); - if (expected !== undefined) { - requireInterruptedSelection( - expected, - interruptedSelection(lock, original) - ); - } - await requireDeadOwner(owner); - await check(); - await recheckDirectories([recovery, lock]); - const latest = await readPrivate(ownerPath, LOCK_OWNER_LIMIT); - if ( - !sameFile(original.info, latest.info) || - latest.text !== original.text || - JSON.stringify(await readdir(lockPath)) !== '["owner"]' - ) { - return refuse(); - } - await requireDeadOwner(owner); - await recheckDirectories([recovery, lock]); - await unlink(ownerPath); - await rmdir(lockPath); - await parent?.file.sync(); - } finally { - await lock.file.close(); - } + await retireInterruptedLock(recovery, expected); } finally { try { await check(); diff --git a/tests/native-private-lock-recovery.test.ts b/tests/native-private-lock-recovery.test.ts index 98a9e3f3f..737193cf8 100644 --- a/tests/native-private-lock-recovery.test.ts +++ b/tests/native-private-lock-recovery.test.ts @@ -1,4 +1,5 @@ -import { afterEach, expect, test } from "bun:test"; +import { afterEach, expect, spyOn, test } from "bun:test"; +import { chmodSync, copyFileSync, renameSync } from "node:fs"; import { chmod, copyFile, @@ -36,7 +37,7 @@ afterEach(async () => { } }); -async function fixture() { +async function fixture(check = () => Promise.resolve()) { const root = await realpath( await mkdtemp(join(tmpdir(), "native-dead-lock-")) ); @@ -75,7 +76,10 @@ async function fixture() { lockPath: join(root, "held"), recoveryPath: join(root, "recovering"), parent, - check: () => recheckDirectories([parent]), + check: async () => { + await check(); + await recheckDirectories([parent]); + }, }); return { root, @@ -109,6 +113,57 @@ test("selected private lock recovery refuses a live owner and reads a dead owner await current.lock.recoverInterruptedLock(); }); +test("selected retirement rechecks file and caller authority after its final process inspection", async () => { + for (const interference of ["replacement", "cancellation"]) { + let canceled = false; + const current = await fixture(() => { + if (canceled) { + throw new Error("Caller authority is no longer held."); + } + return Promise.resolve(); + }); + await current.kill(); + const selected = await current.lock.selectInterruptedLock(); + const originalSpawn = Bun.spawn; + let inspections = 0; + const intercept = spyOn(Bun, "spawn").mockImplementation((...args) => { + const command = args[0]; + if ( + Array.isArray(command) && + command[0] === "/bin/ps" && + command[2] === String(current.child.pid) + ) { + inspections += 1; + if (inspections === 4) { + if (interference === "replacement") { + const path = join(current.root, "held/owner"); + copyFileSync(path, join(current.root, "replacement")); + chmodSync(join(current.root, "replacement"), 0o600); + renameSync(join(current.root, "replacement"), path); + } else { + canceled = true; + } + } + } + return originalSpawn(...args); + }); + try { + await expect( + current.lock.recoverSelectedInterruptedLock(selected) + ).rejects.toThrow(); + expect(inspections).toBe(4); + expect(await readdir(join(current.root, "held"))).toEqual(["owner"]); + if (interference === "replacement") { + expect((await lstat(join(current.root, "held/owner"))).ino).not.toBe( + selected.file.ino + ); + } + } finally { + intercept.mockRestore(); + } + } +}); + test("selected private lock recovery captures the complete caller selection before its first await", async () => { const current = await fixture(); await current.kill(); From ef1e807af0ee74aeab74a6b7a3cfaa494b067ce8 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 15:07:18 -0400 Subject: [PATCH 19/47] feat: bound explicit native recovery request transport --- src/backends/native-runtime-client.ts | 39 ++++++++- tests/native-runtime-client.test.ts | 117 ++++++++++++++++++++++++-- 2 files changed, 150 insertions(+), 6 deletions(-) diff --git a/src/backends/native-runtime-client.ts b/src/backends/native-runtime-client.ts index 9ba8f6017..0f8ffee83 100644 --- a/src/backends/native-runtime-client.ts +++ b/src/backends/native-runtime-client.ts @@ -8,6 +8,7 @@ import { const ERROR_CODE = /^[a-z][a-z0-9_]{0,63}$/; const NATIVE_RUN = /^[a-f0-9]{32}$/; +const NATIVE_HASH = /^[a-f0-9]{64}$/; interface NativeFailure { readonly code: string; readonly causeCode?: string; @@ -77,6 +78,8 @@ export async function invokeNativeRuntime(opts: { readonly boundNativeStatusDrain?: boolean; /** Native source planning and journal inspection also bind pipe lifetime to their owned read child. */ readonly boundNativeAuthoredReadDrain?: boolean; + /** Explicit recovery requests have their own closed argv domain and never accept private input. */ + readonly boundNativeAuthoredRecoveryDrain?: "selection" | "cleanup"; }): Promise { if (opts.signal?.aborted) { throw new NativeRuntimeRequestError({ @@ -94,6 +97,11 @@ export async function invokeNativeRuntime(opts: { opts.boundNativeAuthoredReadDrain, opts.privateInput !== undefined ) && + validNativeAuthoredRecoveryDrainSelection( + opts.args, + opts.boundNativeAuthoredRecoveryDrain, + opts.privateInput !== undefined + ) && Number.isSafeInteger(timeoutMs) ) || timeoutMs < 1 || @@ -120,7 +128,9 @@ export async function invokeNativeRuntime(opts: { } ); const drain = - opts.boundNativeStatusDrain || opts.boundNativeAuthoredReadDrain + opts.boundNativeStatusDrain || + opts.boundNativeAuthoredReadDrain || + opts.boundNativeAuthoredRecoveryDrain ? new AbortController() : undefined; let drainTimer: ReturnType | undefined; @@ -223,6 +233,33 @@ function validNativeAuthoredReadDrainSelection( ); } +function validNativeAuthoredRecoveryDrainSelection( + args: readonly string[], + selected: "selection" | "cleanup" | undefined, + privateInput: boolean +): boolean { + return ( + selected === undefined || + (!privateInput && + args[0] === "graph" && + args[1] === "native" && + args[3] === "--run-id" && + NATIVE_RUN.test(args[4] ?? "") && + ((selected === "selection" && + args.length === 6 && + args[2] === "recovery-selection" && + args[5] === "--json") || + (selected === "cleanup" && + args.length === 10 && + args[2] === "recover-live-owner" && + args[5] === "--expect-receipt" && + NATIVE_HASH.test(args[6] ?? "") && + args[7] === "--expect-owner" && + NATIVE_HASH.test(args[8] ?? "") && + args[9] === "--json"))) + ); +} + function nativeFailureMessage(failure: NativeFailure | undefined): string { const code = failure ? ` (${failure.code}${failure.causeCode ? `: ${failure.causeCode}` : ""})` diff --git a/tests/native-runtime-client.test.ts b/tests/native-runtime-client.test.ts index c98239cce..865902c84 100644 --- a/tests/native-runtime-client.test.ts +++ b/tests/native-runtime-client.test.ts @@ -176,10 +176,97 @@ test("native authored read draining admits only exact public plan and inspect re ).rejects.toThrow("budget"); }); +test("native recovery draining requires exact independent selectors and refuses private stdin before spawn", async () => { + const runtime = await fixture(); + const run = "a".repeat(32); + const selection = [ + "graph", + "native", + "recovery-selection", + "--run-id", + run, + "--json", + ]; + const cleanup = [ + "graph", + "native", + "recover-live-owner", + "--run-id", + run, + "--expect-receipt", + "b".repeat(64), + "--expect-owner", + "c".repeat(64), + "--json", + ]; + for (const [mode, args] of [ + ["selection", selection], + ["cleanup", cleanup], + ] as const) { + expect( + await invokeNativeRuntime({ + runtime, + cwd: runtime.home, + args, + boundNativeAuthoredRecoveryDrain: mode, + }) + ).toMatchObject({ + args: ["--candidate-root", runtime.home, ...args], + inputBytes: 0, + }); + } + const absent = { + home: "/absent-native-recovery-home", + binary: "/absent-native-recovery-binary", + }; + for (const [mode, args] of [ + ["cleanup", selection], + ["selection", cleanup], + ["selection", [...selection, "--json"]], + ["selection", ["graph", "recovery-selection", "--run-id", run, "--json"]], + [ + "cleanup", + cleanup.map((arg) => (arg === "--expect-owner" ? "--source-file" : arg)), + ], + [ + "cleanup", + cleanup.map((arg) => (arg === "c".repeat(64) ? "../foreign" : arg)), + ], + ["cleanup", cleanup.map((arg) => (arg === run ? run.toUpperCase() : arg))], + ] as const) { + await expect( + invokeNativeRuntime({ + runtime: absent, + cwd: absent.home, + args, + boundNativeAuthoredRecoveryDrain: mode, + }) + ).rejects.toThrow("budget"); + } + for (const [mode, args] of [ + ["selection", selection], + ["cleanup", cleanup], + ] as const) { + await expect( + invokeNativeRuntime({ + runtime: absent, + cwd: absent.home, + args, + privateInput: new Uint8Array(), + boundNativeAuthoredRecoveryDrain: mode, + }) + ).rejects.toThrow("budget"); + } +}); + test.each([ - "exit", - "abort", -] as const)("authored read %s finishes while a descendant retains stdout and stderr", async (mode) => { + ["read", "exit"], + ["read", "abort"], + ["selection", "exit"], + ["selection", "abort"], + ["cleanup", "exit"], + ["cleanup", "abort"], +] as const)("authored %s %s finishes while a descendant retains stdout and stderr", async (selection, mode) => { const runtime = await fixture(); const holding = join(runtime.home, "holding"); const heartbeat = join(runtime.home, "heartbeat"); @@ -202,13 +289,33 @@ ${mode === "abort" ? "await Bun.sleep(60_000);" : "process.exit(0);"} ); const controller = new AbortController(); const started = performance.now(); + const run = "a".repeat(32); + const args = + selection === "read" + ? ["graph", "native", "inspect", "--run-id", run, "--json"] + : selection === "selection" + ? ["graph", "native", "recovery-selection", "--run-id", run, "--json"] + : [ + "graph", + "native", + "recover-live-owner", + "--run-id", + run, + "--expect-receipt", + "b".repeat(64), + "--expect-owner", + "c".repeat(64), + "--json", + ]; const request = invokeNativeRuntime({ runtime, cwd: runtime.home, - args: ["graph", "native", "inspect", "--run-id", "a".repeat(32), "--json"], + args, timeoutMs: 5000, signal: controller.signal, - boundNativeAuthoredReadDrain: true, + boundNativeAuthoredReadDrain: selection === "read", + boundNativeAuthoredRecoveryDrain: + selection === "read" ? undefined : selection, }); void request.catch(() => undefined); let failed: unknown; From 436fd367330709f693b9f926e7ca28452408d257 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 15:10:45 -0400 Subject: [PATCH 20/47] test: preserve Bun spawn overload in recovery inspection control --- tests/native-private-lock-recovery.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/native-private-lock-recovery.test.ts b/tests/native-private-lock-recovery.test.ts index 737193cf8..4f6096103 100644 --- a/tests/native-private-lock-recovery.test.ts +++ b/tests/native-private-lock-recovery.test.ts @@ -145,7 +145,9 @@ test("selected retirement rechecks file and caller authority after its final pro } } } - return originalSpawn(...args); + return Array.isArray(command) + ? originalSpawn(command, args[1]) + : originalSpawn(command); }); try { await expect( From 476c09fd48fb15c5c3b1c76d4d5252f621cbdf9b Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 15:15:16 -0400 Subject: [PATCH 21/47] test: forward recovery inspection spawn through exact overload --- tests/native-private-lock-recovery.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/native-private-lock-recovery.test.ts b/tests/native-private-lock-recovery.test.ts index 4f6096103..5946b8f2f 100644 --- a/tests/native-private-lock-recovery.test.ts +++ b/tests/native-private-lock-recovery.test.ts @@ -145,9 +145,12 @@ test("selected retirement rechecks file and caller authority after its final pro } } } - return Array.isArray(command) - ? originalSpawn(command, args[1]) - : originalSpawn(command); + const child: ReturnType = Reflect.apply( + originalSpawn, + Bun, + args + ); + return child; }); try { await expect( From 77af0ca53af94ab921b4f93f8ccc7768e621018f Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 15:17:20 -0400 Subject: [PATCH 22/47] feat: retain issued frontend recovery lease through retirement --- src/lib/native-compose-private-state.ts | 99 +++++++++++++++-- tests/native-private-lock-recovery.test.ts | 122 +++++++++++++++++++++ 2 files changed, 213 insertions(+), 8 deletions(-) diff --git a/src/lib/native-compose-private-state.ts b/src/lib/native-compose-private-state.ts index a59a2377a..c08591f00 100644 --- a/src/lib/native-compose-private-state.ts +++ b/src/lib/native-compose-private-state.ts @@ -32,8 +32,11 @@ export type NativeComposeMutationLease = { readonly token: string; readonly directory: { readonly dev: number; readonly ino: number }; readonly owner: { readonly dev: number; readonly ino: number }; + /** Private copied owner/inode metadata; this snapshot alone is not a lease. */ + readonly selection: NativeComposeInterruptedLockSelection; readonly assertHeld: () => Promise; }; +const mutationLeases = new WeakSet(); type LockOwner = { readonly version: 1; readonly token: string; @@ -585,14 +588,18 @@ export function createNativeComposePrivateMutationLock(opts: { ownerInfo = await writeExclusive(ownerPath, lockToken); await lock.file.sync(); await assertHeld(); - return await run( - Object.freeze({ - token: lockOwner.token, - directory: Object.freeze({ dev: lock.info.dev, ino: lock.info.ino }), - owner: Object.freeze({ dev: ownerInfo.dev, ino: ownerInfo.ino }), - assertHeld, - }) - ); + const lease = Object.freeze({ + token: lockOwner.token, + directory: Object.freeze({ dev: lock.info.dev, ino: lock.info.ino }), + owner: Object.freeze({ dev: ownerInfo.dev, ino: ownerInfo.ino }), + selection: interruptedSelection(lock, { + info: ownerInfo, + text: lockToken, + }), + assertHeld, + }); + mutationLeases.add(lease); + return await run(lease); } finally { try { await assertHeld(); @@ -733,5 +740,81 @@ export function createNativeComposePrivateMutationLock(opts: { const expected = parseNativeComposeInterruptedLockSelection(value); return recoverInterruptedLock(expected); }, + /** Retain the actual named recovery lease through caller-owned durable retirement. + * Expected owner absence is legal only after the caller committed its exact + * retirement intent. Neither a selector nor a fabricated lease grants this API. */ + retireSelectedUnderRecoveryLease(opts: { + readonly selected: unknown; + readonly lease: NativeComposeMutationLease; + readonly allowOwnerAbsent?: boolean; + }) { + const expected = parseNativeComposeInterruptedLockSelection( + opts.selected + ); + const lease = opts.lease; + const allowOwnerAbsent = opts.allowOwnerAbsent ?? false; + if (!mutationLeases.has(lease) || typeof allowOwnerAbsent !== "boolean") { + return refuse(); + } + return (async () => { + await lease.assertHeld(); + const recovery = await holdDirectory(recoveryPath, true); + try { + if (!sameFile(recovery.info, lease.directory)) { + return refuse(); + } + const verify = async () => { + await lease.assertHeld(); + await check(); + await recheckDirectories([recovery]); + }; + await verify(); + const lock = await holdDirectory(lockPath, true); + try { + if (!sameFile(lock.info, expected.directory)) { + return refuse(); + } + const ownerPath = join(lockPath, "owner"); + await requireDeadOwner(expected.owner); + await verify(); + await recheckDirectories([lock]); + const names = await readdir(lockPath); + if ( + JSON.stringify(names) !== '["owner"]' && + !(allowOwnerAbsent && names.length === 0) + ) { + return refuse(); + } + const current = await readPrivate( + ownerPath, + LOCK_OWNER_LIMIT + ).catch((error: unknown) => { + if (allowOwnerAbsent && hasCode(error, "ENOENT")) { + return undefined; + } + throw error; + }); + if (current !== undefined) { + requireInterruptedSelection( + expected, + interruptedSelection(lock, current) + ); + await unlink(ownerPath); + } + await verify(); + await recheckDirectories([lock]); + if ((await readdir(lockPath)).length !== 0) { + return refuse(); + } + await rmdir(lockPath); + await parent?.file.sync(); + } finally { + await lock.file.close(); + } + } finally { + await recovery.file.close(); + } + })(); + }, }; } diff --git a/tests/native-private-lock-recovery.test.ts b/tests/native-private-lock-recovery.test.ts index 5946b8f2f..949dedb88 100644 --- a/tests/native-private-lock-recovery.test.ts +++ b/tests/native-private-lock-recovery.test.ts @@ -243,3 +243,125 @@ test("private dead-lock selectors close nested fields and do not admit another h current.lock.recoverSelectedInterruptedLock(original) ).rejects.toThrow(); }); + +test("durable frontend retirement retains the issued named recovery lease and refuses a copied or foreign lease", async () => { + const current = await fixture(); + await current.kill(); + const selected = await current.lock.selectInterruptedLock(); + const parent = await holdDirectory(current.root, true); + try { + const owner = (name: string) => + createNativeComposePrivateMutationLock({ + lockPath: join(current.root, name), + recoveryPath: join(current.root, `${name}.recovery`), + parent, + check: () => recheckDirectories([parent]), + }); + await owner("foreign").withLock(async (lease) => { + await expect( + current.lock.retireSelectedUnderRecoveryLease({ selected, lease }) + ).rejects.toThrow(); + expect(await readdir(join(current.root, "held"))).toEqual(["owner"]); + }); + await owner("recovering").withLock(async (lease) => { + expect(() => + current.lock.retireSelectedUnderRecoveryLease({ + selected, + lease: { ...lease }, + }) + ).toThrow(); + await current.lock.retireSelectedUnderRecoveryLease({ selected, lease }); + await lease.assertHeld(); + expect((await readdir(current.root)).includes("held")).toBe(false); + expect(await readdir(join(current.root, "recovering"))).toEqual([ + "owner", + ]); + }); + expect(await readdir(current.root)).toEqual(["ready"]); + } finally { + await parent.file.close(); + } +}); + +test("owned recovery lease permits missing owner only at caller's committed retirement phase", async () => { + const current = await fixture(); + await current.kill(); + const selected = await current.lock.selectInterruptedLock(); + const parent = await holdDirectory(current.root, true); + try { + const recovery = createNativeComposePrivateMutationLock({ + lockPath: join(current.root, "recovering"), + recoveryPath: join(current.root, "recovering.recovery"), + parent, + check: () => recheckDirectories([parent]), + }); + await recovery.withLock(async (lease) => { + await unlink(join(current.root, "held/owner")); + await expect( + current.lock.retireSelectedUnderRecoveryLease({ selected, lease }) + ).rejects.toThrow(); + expect(await readdir(join(current.root, "held"))).toEqual([]); + await current.lock.retireSelectedUnderRecoveryLease({ + selected, + lease, + allowOwnerAbsent: true, + }); + await lease.assertHeld(); + }); + expect(await readdir(current.root)).toEqual(["ready"]); + } finally { + await parent.file.close(); + } +}); + +test("held recovery retirement refuses guard loss during its dead-process inspection", async () => { + let changed = false; + const current = await fixture(() => { + if (changed) { + throw new Error("Retained caller authority changed."); + } + return Promise.resolve(); + }); + await current.kill(); + const selected = await current.lock.selectInterruptedLock(); + const parent = await holdDirectory(current.root, true); + try { + const recovery = createNativeComposePrivateMutationLock({ + lockPath: join(current.root, "recovering"), + recoveryPath: join(current.root, "recovering.recovery"), + parent, + check: () => recheckDirectories([parent]), + }); + await recovery.withLock(async (lease) => { + const actual = Bun.spawn; + const intercept = spyOn(Bun, "spawn").mockImplementation((...args) => { + const command = args[0]; + if ( + Array.isArray(command) && + command[0] === "/bin/ps" && + command[2] === String(current.child.pid) + ) { + changed = true; + } + const child: ReturnType = Reflect.apply( + actual, + Bun, + args + ); + return child; + }); + try { + await expect( + current.lock.retireSelectedUnderRecoveryLease({ selected, lease }) + ).rejects.toThrow(); + expect(changed).toBe(true); + expect(await readdir(join(current.root, "held"))).toEqual(["owner"]); + await lease.assertHeld(); + } finally { + intercept.mockRestore(); + } + }); + } finally { + await parent.file.close(); + } +}); From e3a2a92b93e045c9b5f81f03be2bfbdcd0471152 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 15:22:53 -0400 Subject: [PATCH 23/47] fix: revoke frontend recovery lease before final deletion --- src/lib/native-compose-private-state.ts | 14 ++++ tests/native-private-lock-recovery.test.ts | 74 ++++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/src/lib/native-compose-private-state.ts b/src/lib/native-compose-private-state.ts index c08591f00..b915b65c7 100644 --- a/src/lib/native-compose-private-state.ts +++ b/src/lib/native-compose-private-state.ts @@ -34,6 +34,7 @@ export type NativeComposeMutationLease = { readonly owner: { readonly dev: number; readonly ino: number }; /** Private copied owner/inode metadata; this snapshot alone is not a lease. */ readonly selection: NativeComposeInterruptedLockSelection; + readonly assertActive: () => undefined; readonly assertHeld: () => Promise; }; const mutationLeases = new WeakSet(); @@ -566,6 +567,7 @@ export function createNativeComposePrivateMutationLock(opts: { const ownerPath = join(lockPath, "owner"); const lockToken = JSON.stringify(lockOwner); let ownerInfo: Stats | undefined; + let issued: NativeComposeMutationLease | undefined; let active = true; const assertHeld = async () => { if (!active) { @@ -596,11 +598,21 @@ export function createNativeComposePrivateMutationLock(opts: { info: ownerInfo, text: lockToken, }), + assertActive: () => { + if (issued === undefined || !mutationLeases.has(issued) || !active) { + return refuse(); + } + return undefined; + }, assertHeld, }); mutationLeases.add(lease); + issued = lease; return await run(lease); } finally { + if (issued !== undefined) { + mutationLeases.delete(issued); + } try { await assertHeld(); await unlink(ownerPath); @@ -799,6 +811,7 @@ export function createNativeComposePrivateMutationLock(opts: { expected, interruptedSelection(lock, current) ); + lease.assertActive(); await unlink(ownerPath); } await verify(); @@ -806,6 +819,7 @@ export function createNativeComposePrivateMutationLock(opts: { if ((await readdir(lockPath)).length !== 0) { return refuse(); } + lease.assertActive(); await rmdir(lockPath); await parent?.file.sync(); } finally { diff --git a/tests/native-private-lock-recovery.test.ts b/tests/native-private-lock-recovery.test.ts index 949dedb88..e238e7975 100644 --- a/tests/native-private-lock-recovery.test.ts +++ b/tests/native-private-lock-recovery.test.ts @@ -1,5 +1,6 @@ import { afterEach, expect, spyOn, test } from "bun:test"; import { chmodSync, copyFileSync, renameSync } from "node:fs"; +import * as privateFiles from "node:fs/promises"; import { chmod, copyFile, @@ -365,3 +366,76 @@ test("held recovery retirement refuses guard loss during its dead-process inspec await parent.file.close(); } }); + +test("retirement cannot unlink the selected owner when its recovery callback settles during the final read", async () => { + const current = await fixture(); + await current.kill(); + const selected = await current.lock.selectInterruptedLock(); + const parent = await holdDirectory(current.root, true); + let readReached!: () => void; + const reached = new Promise((resolve) => { + readReached = resolve; + }); + let finishRead!: () => void; + const paused = new Promise((resolve) => { + finishRead = resolve; + }); + const originalOpen = privateFiles.open; + const readers: ReturnType[] = []; + const intercept = spyOn(privateFiles, "open").mockImplementation( + async (...args) => { + const file: Awaited> = + await Reflect.apply(originalOpen, privateFiles, args); + if (args[0] === join(current.root, "held/owner")) { + const originalRead = file.read; + readers.push( + spyOn(file, "read").mockImplementation(async (...readArgs) => { + const result = await Reflect.apply(originalRead, file, readArgs); + readReached(); + await paused; + return result; + }) + ); + } + return file; + } + ); + let request: Promise | undefined; + try { + const recovery = createNativeComposePrivateMutationLock({ + lockPath: join(current.root, "recovering"), + recoveryPath: join(current.root, "recovering.recovery"), + parent, + check: () => recheckDirectories([parent]), + }); + await recovery.withLock(async (lease) => { + request = current.lock.retireSelectedUnderRecoveryLease({ + selected, + lease, + }); + void request.catch(() => undefined); + await Promise.race([ + reached, + Bun.sleep(3000).then(() => { + throw new Error("Owned read pause was not reached."); + }), + ]); + // Returning ends the actual issued lease while its retirement is awaiting read. + }); + expect((await readdir(current.root)).includes("recovering")).toBe(false); + finishRead(); + await expect(request).rejects.toThrow(); + expect(await readdir(join(current.root, "held"))).toEqual(["owner"]); + expect((await lstat(join(current.root, "held/owner"))).ino).toBe( + selected.file.ino + ); + } finally { + finishRead(); + await request?.catch(() => undefined); + for (const reader of readers) { + reader.mockRestore(); + } + intercept.mockRestore(); + await parent.file.close(); + } +}); From a429527eafd16f256293c4e65d49ad837b622820 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 15:50:58 -0400 Subject: [PATCH 24/47] feat: add explicit native frontend recovery owner --- docs/reference/native-authored-commands.md | 24 +- .../native-authored-project-recovery.ts | 779 ++++++++++++++++++ src/backends/native-authored-project-run.ts | 115 ++- src/lib/native-authored-command.ts | 70 +- tests/native-authored-command.test.ts | 115 +++ .../native-authored-project-recovery.test.ts | 429 ++++++++++ 6 files changed, 1520 insertions(+), 12 deletions(-) create mode 100644 src/backends/native-authored-project-recovery.ts create mode 100644 tests/native-authored-project-recovery.test.ts diff --git a/docs/reference/native-authored-commands.md b/docs/reference/native-authored-commands.md index 572893451..6644a9147 100644 --- a/docs/reference/native-authored-commands.md +++ b/docs/reference/native-authored-commands.md @@ -22,7 +22,7 @@ and host effects remain outside this bounded frontend. Unsupported intent must refuse before managed value resolution and provider work. Early typed input capability refusals retain `E_NATIVE_PROJECT_UNSUPPORTED` without exposing compiler diagnostics or creating native source/start/run authority. `--detach`, -`--json`, service subsets, recovery and other lifecycle operations refuse before input +`--json`, service subsets and other lifecycle operations refuse before input acquisition. `--branch` supplies an explicit native namespace; its omission uses the canonical project namespace without inferring a Git branch. Profile and env selection keep their compiler contracts. `--env base` bypasses inherited overlays @@ -36,7 +36,27 @@ Ctrl-C requests that shutdown. Exact durable Removed evidence retires the tagged native run and startup intent. Unknown, changed or live cleanup retains the attempt and refuses replay; a missing ready mapping does not authorize a fresh start. +`down --recover` explicitly retires a complete stored native generation whose +foreground publisher and frontend admission owner are both dead on the same host +boot. It requires the current version 3 native publication, the exact stored Ready, +startup intent and private source envelope, and the original guest incarnation. +Live owners, older version 2 dead publications, partial startup, pending writes, +changed files and rebooted guests refuse. Env/profile changes, service subsets and +`--json` are unsupported for this operation. It neither recompiles authored input +nor acquires managed values or starts a provider. + +Recovery commits a distinct private intent before calling the Rust cleanup owner. +The original raw selectors and resource inventory remain fixed through retries; +current cleanup phases may advance. Before retiring each frontend file, the owner +inspects the exact native receipt and requires all resources Removed with null +observations. A committed retirement phase alone permits the corresponding file's +expected absence. Pending writes and replacements remain retained refusals. The +recovery lease blocks ordinary startup throughout this work. Completed frontend +history is archived under fresh startup admission before a later generation can +start; incomplete history never permits automatic takeover. Persistent data stays +retained, and no recovery operation reboots the guest or allocates new resources. + The native receipt and source paths are distinct from strict Compose v1 artifacts. No native hash substitutes for a normalized Compose hash. Source and fake-driver checks do not qualify an installed frontend, a live provider, the full authored -corpus, interrupted recovery or resource overhead; those remain separate gates. +corpus, actual version 3 dead-owner recovery or resource overhead; those remain separate gates. diff --git a/src/backends/native-authored-project-recovery.ts b/src/backends/native-authored-project-recovery.ts new file mode 100644 index 000000000..54688bd08 --- /dev/null +++ b/src/backends/native-authored-project-recovery.ts @@ -0,0 +1,779 @@ +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { link, lstat, open, rename, unlink } from "node:fs/promises"; +import { join } from "node:path"; +import { isRecord } from "../lib/guards.ts"; +import { + createNativeComposePrivateMutationLock, + hasCode, + keys, + type NativeComposeInterruptedLockSelection, + type NativeComposeMutationLease, + parseNativeComposeInterruptedLockSelection, + readPrivate, + sameFile, + synchronizeDirectories, + writeExclusive, +} from "../lib/native-compose-private-state.ts"; +import { + type NativeAuthoredReceipt, + parseNativeAuthoredSnapshot, +} from "./native-authored-graph-protocol.ts"; +import { + type NativeAuthoredProjectRunScope, + type NativeAuthoredProjectRunSelection, + type NativeAuthoredProjectStartSelection, + parseNativeAuthoredProjectRunSelection, + parseNativeAuthoredProjectStartSelection, + withNativeAuthoredProjectRecoveryStorage, +} from "./native-authored-project-run.ts"; +import { + type NativeAuthoredRecoveryResult, + type NativeAuthoredRecoverySelection, + parseNativeAuthoredRecoveryResult, + parseNativeAuthoredRecoverySelection, +} from "./native-authored-recovery-protocol.ts"; +import { + invokeNativeRuntime, + type NativeRuntimeSelection, +} from "./native-runtime-client.ts"; + +const LIMIT = 64 * 1024; +const HASH = /^[a-f0-9]{64}$/; +const PHASES = [ + "prepared", + "native-removed", + "ready-retirement", + "ready-retired", + "start-retirement", + "start-retired", + "source-retirement", + "source-retired", + "admission-retirement", + "complete", +] as const; +type Phase = (typeof PHASES)[number]; +type Identity = NativeAuthoredProjectRunSelection["identity"]; +type Intent = { + readonly version: 1; + readonly kind: "native-authored-project-recovery"; + readonly phase: Phase; + readonly ready: NativeAuthoredProjectRunSelection; + readonly start: NativeAuthoredProjectStartSelection; + readonly source: Identity; + readonly admission: NativeComposeInterruptedLockSelection; + readonly native: NativeAuthoredRecoverySelection; + readonly lease: NativeComposeInterruptedLockSelection; + readonly mutation: NativeComposeInterruptedLockSelection; + readonly lease_releasing: boolean; + readonly mutation_retirement: boolean; +}; +type Store = Parameters< + Parameters[1] +>[0]; +type Saved = { readonly record: Intent; readonly identity: Identity }; +function refused(): never { + throw new Error( + "Native frontend recovery is retained or changed; values omitted. No startup was replayed." + ); +} +function hash(text: string): string { + return createHash("sha256").update(text).digest("hex"); +} +function copyIdentity(value: unknown): Identity { + if ( + !(isRecord(value) && keys(value, "dev,ino,sha256")) || + typeof value.dev !== "number" || + !Number.isSafeInteger(value.dev) || + value.dev < 0 || + typeof value.ino !== "number" || + !Number.isSafeInteger(value.ino) || + value.ino < 1 || + typeof value.sha256 !== "string" || + !HASH.test(value.sha256) + ) { + return refused(); + } + return { dev: value.dev, ino: value.ino, sha256: value.sha256 }; +} +function freeze(value: T): T { + if (value !== null && typeof value === "object") { + for (const item of Object.values(value)) { + freeze(item); + } + Object.freeze(value); + } + return value; +} +function parseIntent(value: unknown): Intent { + if ( + !( + isRecord(value) && + keys( + value, + "admission,kind,lease,lease_releasing,mutation,mutation_retirement,native,phase,ready,source,start,version" + ) + ) || + value.version !== 1 || + value.kind !== "native-authored-project-recovery" || + typeof value.phase !== "string" || + !PHASES.includes(value.phase as Phase) || + typeof value.lease_releasing !== "boolean" || + typeof value.mutation_retirement !== "boolean" + ) { + return refused(); + } + const ready = parseNativeAuthoredProjectRunSelection(value.ready); + const start = parseNativeAuthoredProjectStartSelection(value.start); + const admission = parseNativeComposeInterruptedLockSelection(value.admission); + const lease = parseNativeComposeInterruptedLockSelection(value.lease); + const mutation = parseNativeComposeInterruptedLockSelection(value.mutation); + if ( + JSON.stringify(start.record.review) !== + JSON.stringify(ready.record.receipt.review) || + lease.owner.bootId !== admission.owner.bootId || + mutation.owner.bootId !== admission.owner.bootId + ) { + return refused(); + } + return freeze({ + version: 1, + kind: "native-authored-project-recovery", + phase: value.phase as Phase, + ready, + start, + source: copyIdentity(value.source), + admission, + native: parseNativeAuthoredRecoverySelection({ + value: value.native, + admitted: ready.record.receipt, + }), + lease, + mutation, + lease_releasing: value.lease_releasing, + mutation_retirement: value.mutation_retirement, + }); +} +function rank(record: Intent): number { + return PHASES.indexOf(record.phase); +} +function phaseAt(index: number): Phase { + return PHASES[index] ?? refused(); +} +async function absent(path: string): Promise { + try { + await lstat(path); + return false; + } catch (error) { + if (hasCode(error, "ENOENT")) { + return true; + } + throw error; + } +} +function intentPath(store: Store) { + return `${store.ready}.recovery.json`; +} +function pendingPath(store: Store) { + return `${intentPath(store)}.pending`; +} +async function linkedArchive( + store: Store, + archive: string, + expected: Identity +): Promise { + const file = await open( + intentPath(store), + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK + ); + try { + const before = await file.stat(); + if ( + !before.isFile() || + before.uid !== process.getuid?.() || + (before.mode & 0o777) !== 0o600 || + before.nlink !== 2 || + before.size < 1 || + before.size > LIMIT || + !sameFile(before, expected) + ) { + return refused(); + } + const bytes = Buffer.alloc(before.size + 1); + const { bytesRead } = await file.read(bytes, 0, bytes.length, 0); + const after = await file.stat(); + const original = await lstat(intentPath(store)); + const copied = await lstat(archive); + if ( + bytesRead !== before.size || + !sameFile(after, before) || + after.size !== before.size || + after.mtimeMs !== before.mtimeMs || + after.ctimeMs !== before.ctimeMs || + !sameFile(original, before) || + !sameFile(copied, before) || + original.nlink !== 2 || + copied.nlink !== 2 || + createHash("sha256") + .update(bytes.subarray(0, bytesRead)) + .digest("hex") !== expected.sha256 + ) { + return refused(); + } + } finally { + await file.close(); + } +} +async function readIntent(store: Store): Promise { + if (!(await absent(pendingPath(store)))) { + return refused(); + } + let read: Awaited>; + try { + read = await readPrivate(intentPath(store), LIMIT); + } catch (error) { + if (hasCode(error, "ENOENT")) { + return null; + } + throw error; + } + const value: unknown = JSON.parse(read.text); + if ( + !(isRecord(value) && keys(value, "record,scope")) || + value.scope !== store.scope + ) { + return refused(); + } + return { + record: parseIntent(value.record), + identity: { + dev: read.info.dev, + ino: read.info.ino, + sha256: hash(read.text), + }, + }; +} +function lock(store: Store, path: string, recoveryPath: string) { + return createNativeComposePrivateMutationLock({ + lockPath: path, + recoveryPath, + parent: store.root, + check: store.check, + }); +} +async function expectedFile( + path: string, + expected: Identity, + optional: boolean +): Promise { + const read = await readPrivate(path, 1024 * 1024).catch((error: unknown) => { + if (optional && hasCode(error, "ENOENT")) { + return undefined; + } + throw error; + }); + if (read === undefined) { + return false; + } + if (!sameFile(read.info, expected) || hash(read.text) !== expected.sha256) { + return refused(); + } + return true; +} +async function bindings(store: Store, record: Intent): Promise { + const phase = rank(record); + for (const [path, expected, retirement] of [ + [store.ready, record.ready.identity, 2], + [store.start, record.start.identity, 4], + [store.sourcePath(record.native.run), record.source, 6], + ] as const) { + if (phase > retirement) { + if (!(await absent(path))) { + return refused(); + } + } else { + await expectedFile(path, expected, phase === retirement); + } + } + if (phase < 8) { + const selected = await lock( + store, + store.admission, + `${store.admission}.verification` + ).selectInterruptedLock(); + if (JSON.stringify(selected) !== JSON.stringify(record.admission)) { + return refused(); + } + } else if (phase === 9 && !(await absent(store.admission))) { + return refused(); + } +} + +type Retirement = { + readonly record: () => Intent; + readonly commit: (record: Intent) => Promise; + readonly check: () => Promise; + readonly active: () => undefined; + readonly inspect: () => Promise; +}; +async function retireFiles( + store: Store, + retirement: Retirement +): Promise { + const selected = retirement.record(); + for (const [path, expected, phase] of [ + [store.ready, selected.ready.identity, 2], + [store.start, selected.start.identity, 4], + [store.sourcePath(selected.native.run), selected.source, 6], + ] as const) { + if (rank(retirement.record()) < phase) { + await retirement.commit({ + ...retirement.record(), + phase: phaseAt(phase), + }); + } + if (rank(retirement.record()) !== phase) { + continue; + } + await retirement.inspect(); + await retirement.check(); + if (await expectedFile(path, expected, true)) { + retirement.active(); + await unlink(path); + await synchronizeDirectories(store.held); + } + if (!(await absent(path))) { + return refused(); + } + await retirement.commit({ + ...retirement.record(), + phase: phaseAt(phase + 1), + }); + } +} +async function retireAdmission( + store: Store, + retirement: Retirement, + lease: NativeComposeMutationLease +): Promise { + if (rank(retirement.record()) < 8) { + await retirement.commit({ + ...retirement.record(), + phase: "admission-retirement", + }); + } + if (rank(retirement.record()) !== 8) { + return; + } + await retirement.inspect(); + await retirement.check(); + if (!(await absent(store.admission))) { + await lock( + store, + store.admission, + store.recovery + ).retireSelectedUnderRecoveryLease({ + selected: retirement.record().admission, + lease, + allowOwnerAbsent: true, + }); + } + await retirement.check(); + if (!(await absent(store.admission))) { + return refused(); + } + await retirement.commit({ ...retirement.record(), phase: "complete" }); +} + +/** Explicit stored-generation cleanup only. No compilation, value acquisition, + * serve, allocation or automatic startup takeover exists in this owner. */ +export async function recoverNativeAuthoredProject(opts: { + readonly scope: NativeAuthoredProjectRunScope; + readonly runtime: NativeRuntimeSelection; + readonly timeoutMs: number; + readonly signal?: AbortSignal; + /** Exact public native-request seam for isolated owner tests. */ + readonly request?: typeof invokeNativeRuntime; +}): Promise { + const runtime = { ...opts.runtime }; + const timeoutMs = opts.timeoutMs; + const signal = opts.signal; + const request = opts.request ?? invokeNativeRuntime; + if ( + !Number.isSafeInteger(timeoutMs) || + timeoutMs < 1 || + timeoutMs > 300_000 || + signal?.aborted || + runtime.home !== opts.scope.nativeHome + ) { + return refused(); + } + const deadline = performance.now() + timeoutMs; + const remaining = () => { + const value = Math.floor(deadline - performance.now()); + if (value < 1 || signal?.aborted) { + return refused(); + } + return value; + }; + return await withNativeAuthoredProjectRecoveryStorage( + opts.scope, + async (store) => { + let saved = await readIntent(store); + const recovery = lock( + store, + store.recovery, + `${store.recovery}.recovery` + ); + if (!(await absent(store.recovery))) { + if (!saved) { + return refused(); + } + const current = await recovery.selectInterruptedLock(); + if (JSON.stringify(current) !== JSON.stringify(saved.record.lease)) { + return refused(); + } + await recovery.recoverSelectedInterruptedLock(current); + } else if (saved && !saved.record.lease_releasing) { + return refused(); + } + return await recovery.withLock(async (lease) => { + const check = async () => { + await lease.assertHeld(); + await store.check(); + const current = await readIntent(store); + if (JSON.stringify(current) !== JSON.stringify(saved)) { + return refused(); + } + }; + const commit = async ( + value: Intent, + mutation?: NativeComposeMutationLease + ) => { + const active = () => { + lease.assertActive(); + mutation?.assertActive(); + }; + const record = parseIntent(value); + const text = JSON.stringify({ scope: store.scope, record }); + if (Buffer.byteLength(text) > LIMIT) { + return refused(); + } + await check(); + await mutation?.assertHeld(); + const info = await writeExclusive(pendingPath(store), text); + // This attempt's own pending file is allowed only at this publication boundary. + await lease.assertHeld(); + await mutation?.assertHeld(); + await store.check(); + const current = await readPrivate(pendingPath(store), LIMIT); + if (!sameFile(current.info, info) || current.text !== text) { + return refused(); + } + if (saved) { + const old = await readPrivate(intentPath(store), LIMIT); + if ( + !sameFile(old.info, saved.identity) || + hash(old.text) !== saved.identity.sha256 + ) { + return refused(); + } + active(); + await rename(pendingPath(store), intentPath(store)); + } else { + active(); + await link(pendingPath(store), intentPath(store)); + active(); + await unlink(pendingPath(store)); + } + await synchronizeDirectories(store.held); + saved = await readIntent(store); + if ( + !saved || + JSON.stringify(saved.record) !== JSON.stringify(record) + ) { + return refused(); + } + }; + const mutation = lock( + store, + store.mutation, + `${store.mutation}.recovery` + ); + if (!(await absent(store.mutation))) { + if (!saved) { + return refused(); + } + if (!saved.record.mutation_retirement) { + const selected = await mutation.selectInterruptedLock(); + if ( + JSON.stringify(selected) !== JSON.stringify(saved.record.mutation) + ) { + return refused(); + } + } + await commit({ + ...saved.record, + lease: lease.selection, + mutation_retirement: true, + }); + await lock( + store, + store.mutation, + store.recovery + ).retireSelectedUnderRecoveryLease({ + selected: saved.record.mutation, + lease, + allowOwnerAbsent: true, + }); + } else if ( + saved && + !saved.record.lease_releasing && + !saved.record.mutation_retirement + ) { + return refused(); + } + return await mutation.withLock(async (mutationLease) => { + const record = () => saved?.record ?? refused(); + const both = async () => { + remaining(); + await check(); + await mutationLease.assertHeld(); + }; + try { + if (saved) { + await commit( + { + ...saved.record, + lease: lease.selection, + mutation: mutationLease.selection, + lease_releasing: false, + mutation_retirement: false, + }, + mutationLease + ); + } else { + const ready = await store.readReady(); + const start = await store.readStart(); + const source = await store.readSource( + ready.record.receipt.review.provenance.run + ); + const admission = await lock( + store, + store.admission, + `${store.admission}.verification` + ).selectInterruptedLock(); + const native = parseNativeAuthoredRecoverySelection({ + value: await request({ + runtime, + cwd: store.projectRoot, + args: [ + "graph", + "native", + "recovery-selection", + "--run-id", + ready.record.receipt.review.provenance.run, + "--json", + ], + timeoutMs: remaining(), + signal, + boundNativeAuthoredRecoveryDrain: "selection", + }), + admitted: ready.record.receipt, + }); + remaining(); + await commit( + { + version: 1, + kind: "native-authored-project-recovery", + phase: "prepared", + ready, + start, + source, + admission, + native, + lease: lease.selection, + mutation: mutationLease.selection, + lease_releasing: false, + mutation_retirement: false, + }, + mutationLease + ); + } + if (!saved) { + return refused(); + } + await both(); + await bindings(store, saved.record); + await both(); + const selected = saved.record.native; + if (saved.record.phase === "prepared") { + parseNativeAuthoredRecoveryResult({ + expected: selected, + value: await request({ + runtime, + cwd: store.projectRoot, + args: [ + "graph", + "native", + "recover-live-owner", + "--run-id", + selected.run, + "--expect-receipt", + selected.receipt_sha256, + "--expect-owner", + selected.owner_sha256, + "--json", + ], + timeoutMs: remaining(), + signal, + boundNativeAuthoredRecoveryDrain: "cleanup", + }), + }); + await both(); + await bindings(store, saved.record); + await both(); + await commit( + { ...saved.record, phase: "native-removed" }, + mutationLease + ); + } + const inspect = async () => { + const snapshot = parseNativeAuthoredSnapshot({ + value: await request({ + runtime, + cwd: store.projectRoot, + args: [ + "graph", + "native", + "inspect", + "--run-id", + selected.run, + "--json", + ], + timeoutMs: remaining(), + signal, + boundNativeAuthoredReadDrain: true, + }), + expectedReview: selected.receipt.review, + admitted: selected.receipt, + }); + if ( + snapshot.receipt.phase !== "removed" || + Object.values(snapshot.observations).some( + (item) => item !== null + ) + ) { + return refused(); + } + await both(); + await bindings(store, record()); + await both(); + return snapshot.receipt; + }; + await inspect(); + const retirement: Retirement = { + record, + commit: (value) => commit(value, mutationLease), + check: both, + active: () => { + remaining(); + lease.assertActive(); + mutationLease.assertActive(); + return undefined; + }, + inspect, + }; + await retireFiles(store, retirement); + await retireAdmission(store, retirement, lease); + const removed = await inspect(); + await bindings(store, saved.record); + await both(); + return { + version: 1, + kind: "native-graph-live-owner-recovered", + run: selected.run, + same_boot: true, + publication_retired: true, + receipt: removed, + }; + } finally { + if (saved) { + await commit( + { ...saved.record, lease_releasing: true }, + mutationLease + ); + } + } + }); + }); + } + ); +} + +/** Startup may archive an authenticated completed frontend intent under its + * fresh admission. Incomplete recovery never becomes automatic takeover. */ +export async function archiveCompletedNativeAuthoredRecovery(opts: { + readonly scope: NativeAuthoredProjectRunScope; + readonly admission: NativeComposeMutationLease; +}): Promise { + const admission = opts.admission; + await withNativeAuthoredProjectRecoveryStorage(opts.scope, async (store) => { + const saved = await readIntent(store); + if (!saved) { + return; + } + if (saved.record.phase !== "complete" || !saved.record.lease_releasing) { + return refused(); + } + const mutation = lock(store, store.mutation, `${store.mutation}.recovery`); + await mutation.withLock(async (lease) => { + await admission.assertHeld(); + await lease.assertHeld(); + await store.check(); + if ( + !( + (await absent(store.recovery)) && + (await absent(store.ready)) && + (await absent(store.start)) && + (await absent(store.sourcePath(saved.record.native.run))) + ) + ) { + return refused(); + } + const current = await readIntent(store); + if (JSON.stringify(current) !== JSON.stringify(saved)) { + return refused(); + } + const info = await lstat(store.admission); + if (!sameFile(info, admission.directory)) { + return refused(); + } + const archive = join( + store.root.path, + `${saved.record.native.run}.frontend-recovered.json` + ); + admission.assertActive(); + lease.assertActive(); + await link(intentPath(store), archive); + // Only this exact archive transition admits nlink=2. Ordinary private + // readers remain single-link and refuse an interrupted archive publication. + await admission.assertHeld(); + await lease.assertHeld(); + await store.check(); + await linkedArchive(store, archive, saved.identity); + admission.assertActive(); + lease.assertActive(); + await unlink(intentPath(store)); + await synchronizeDirectories(store.held); + const completed = await readPrivate(archive, LIMIT); + if ( + !sameFile(completed.info, saved.identity) || + hash(completed.text) !== saved.identity.sha256 || + !(await absent(intentPath(store))) + ) { + return refused(); + } + }); + }); +} diff --git a/src/backends/native-authored-project-run.ts b/src/backends/native-authored-project-run.ts index b45976a0a..2e8d4a80d 100644 --- a/src/backends/native-authored-project-run.ts +++ b/src/backends/native-authored-project-run.ts @@ -167,6 +167,13 @@ function selection(value: unknown, parse: (value: unknown) => T) { function selectedFile(value: unknown): NativeAuthoredProjectRunSelection { return selection(value, selected); } +/** Copied data selectors used by the separately tagged explicit recovery intent. */ +export function parseNativeAuthoredProjectRunSelection(value: unknown) { + return selectedFile(value); +} +export function parseNativeAuthoredProjectStartSelection(value: unknown) { + return selection(value, started); +} function validBranch(branch: string | null): boolean { return ( branch === null || @@ -313,6 +320,7 @@ async function storage( root, held, file, + mutationPath: join(root.path, `${key}.lock`), startFile: join(root.path, `${key}.start.json`), admissionPath: join(root.path, `${key}.admission.lock`), admissionRecovery: join(root.path, `${key}.admission.recovery`), @@ -340,6 +348,7 @@ type Store = { readonly root: HeldDirectory; readonly held: readonly HeldDirectory[]; readonly file: string; + readonly mutationPath: string; readonly startFile: string; readonly admissionPath: string; readonly admissionRecovery: string; @@ -349,6 +358,100 @@ type Store = { action: (assertHeld: () => Promise) => Promise ) => Promise; }; + +/** Internal frontend storage owner. The awaited callback retains the canonical + * directory handles; no Rust journal/publication path is reconstructed here. */ +export async function withNativeAuthoredProjectRecoveryStorage( + opts: NativeAuthoredProjectRunScope, + action: (store: { + readonly scope: string; + readonly projectRoot: string; + readonly root: HeldDirectory; + readonly held: readonly HeldDirectory[]; + readonly ready: string; + readonly start: string; + readonly admission: string; + readonly recovery: string; + readonly mutation: string; + readonly check: () => Promise; + readonly readReady: () => Promise; + readonly readStart: () => Promise; + readonly sourcePath: (run: string) => string; + readonly readSource: (run: string) => Promise<{ + readonly dev: number; + readonly ino: number; + readonly sha256: string; + }>; + }) => Promise +): Promise { + return await storage(opts, false, async (store) => { + if (!store) { + return refused(); + } + const sourcePath = (run: string) => { + if (!HEX32.test(run)) { + return refused(); + } + return join(store.root.path, `${run}.source.json`); + }; + return await action({ + scope: JSON.stringify(store.identity), + projectRoot: store.identity.projectRoot, + root: store.root, + held: store.held, + ready: store.file, + start: store.startFile, + admission: store.admissionPath, + recovery: store.admissionRecovery, + mutation: store.mutationPath, + check: store.check, + readReady: () => read(store), + readStart: () => readRecord(store, store.startFile, started), + sourcePath, + async readSource(run) { + const current = await readPrivate(sourcePath(run), SOURCE_LIMIT); + const value: unknown = JSON.parse(current.text); + if ( + !( + isRecord(value) && + keys( + value, + "branch,env_metadata,kind,overlay,profiles,project,run,version" + ) + ) || + value.version !== 2 || + value.kind !== "native-graph-source" || + value.project !== store.identity.projectRoot || + value.branch !== store.identity.branch || + value.run !== run || + !Array.isArray(value.profiles) || + value.profiles.length > 64 || + value.profiles.some( + (profile: unknown) => + typeof profile !== "string" || !validBranch(profile) + ) || + !( + value.overlay === "inherit" || + value.overlay === "base" || + (isRecord(value.overlay) && + keys(value.overlay, "named") && + typeof value.overlay.named === "string" && + validBranch(value.overlay.named)) + ) || + !parseNativeEnvMetadata(value.env_metadata) + ) { + return refused(); + } + await store.check(); + return { + dev: current.info.dev, + ino: current.info.ino, + sha256: digest(current.text), + }; + }, + }); + }); +} async function read(store: Store): Promise { return await readRecord(store, store.file, selected); } @@ -561,7 +664,17 @@ export async function withNativeAuthoredProjectAdmission( parent: store.root, check: store.check, }); - return await admission.withHeldLock(async (verifyLock) => { + return await admission.withLock(async (admissionLease) => { + const verifyLock = admissionLease.assertHeld; + // A completed explicit recovery is history, while an incomplete or pending + // intent must block startup before compiler or private value acquisition. + const { archiveCompletedNativeAuthoredRecovery } = await import( + "./native-authored-project-recovery.ts" + ); + await archiveCompletedNativeAuthoredRecovery({ + scope: opts, + admission: admissionLease, + }); let active = true; const assertHeld = async () => { if (!active) { diff --git a/src/lib/native-authored-command.ts b/src/lib/native-authored-command.ts index 20295352d..9c15024ec 100644 --- a/src/lib/native-authored-command.ts +++ b/src/lib/native-authored-command.ts @@ -1,6 +1,7 @@ import { randomBytes } from "node:crypto"; import { realpath } from "node:fs/promises"; import { join } from "node:path"; +import { recoverNativeAuthoredProject } from "../backends/native-authored-project-recovery.ts"; import { NativeAuthoredProjectStartError, serveNativeAuthoredProject, @@ -17,10 +18,32 @@ function unsupported(): never { throw new HackCliError({ code: "E_NATIVE_PROJECT_UNSUPPORTED", message: - "Native authored execution currently requires whole-project foreground up on macOS. This request ran no input or runtime operation.", + "Native authored execution requires whole-project foreground up or explicit stored-generation down --recover on macOS. This request ran no input or runtime operation.", }); } +function assertRecoveryOptions(options: NativeComposeCommandOptions): void { + if ( + process.platform !== "darwin" || + options.operation !== "down" || + !options.recover || + options.detach || + options.json || + options.unsupportedOptions || + options.services !== undefined || + options.service !== undefined || + options.command !== undefined || + options.workdir !== undefined || + options.follow !== undefined || + options.tail !== undefined || + options.logFormat !== undefined || + options.profiles !== undefined || + options.overlay !== undefined + ) { + unsupported(); + } +} + function assertForegroundOptions(options: NativeComposeCommandOptions): void { if ( process.platform !== "darwin" || @@ -68,7 +91,8 @@ function startupFailure(error: unknown): number { /** * Explicit native dispatch after exact authored-family and adoption selection. * Compose and omitted selections retain their existing owner. Only foreground up - * delegates to the tagged native lifetime owner; unsupported requests never start + * delegates to the tagged native lifetime owner. Explicit down --recover uses the + * stored generation owner without input acquisition; unsupported requests never start * input acquisition or runtime work. The Rust planner owns capability refusal. */ export async function tryNativeAuthoredCommand(opts: { @@ -77,6 +101,7 @@ export async function tryNativeAuthoredCommand(opts: { readonly env?: Readonly>; /** Execution seam for command-selection controls; the CLI always uses the real owner. */ readonly serve?: typeof serveNativeAuthoredProject; + readonly recover?: typeof recoverNativeAuthoredProject; }): Promise { const sourceEnv = opts.env ?? process.env; const env = { @@ -105,7 +130,13 @@ export async function tryNativeAuthoredCommand(opts: { }; const projectRoot = opts.selected.projectRoot; const serve = opts.serve ?? serveNativeAuthoredProject; - assertForegroundOptions(options); + const recover = opts.recover ?? recoverNativeAuthoredProject; + const recovering = options.operation === "down" && options.recover === true; + if (recovering) { + assertRecoveryOptions(options); + } else { + assertForegroundOptions(options); + } let runtime: ReturnType; try { runtime = resolveNativeRuntimeSelection(env); @@ -129,14 +160,28 @@ export async function tryNativeAuthoredCommand(opts: { process.on("SIGTERM", cancel); try { const root = await realpath(projectRoot); + const scope = { + projectRoot: root, + projectDir: join(root, ".hack"), + nativeHome: runtime.home, + branch: options.instance ?? null, + }; + if (recovering) { + await recover({ + runtime, + scope, + timeoutMs: startupTimeoutMs, + signal: controller.signal, + }); + logger.info({ + message: + "Selected native project recovered; persistent data is retained.", + }); + return 0; + } return await serve({ runtime, - scope: { - projectRoot: root, - projectDir: join(root, ".hack"), - nativeHome: runtime.home, - branch: options.instance ?? null, - }, + scope, run: randomBytes(16).toString("hex"), profiles: options.profiles, overlay: options.overlay, @@ -148,6 +193,13 @@ export async function tryNativeAuthoredCommand(opts: { }, }); } catch (error: unknown) { + if (recovering) { + throw new HackCliError({ + code: "E_LIFECYCLE_FAILED", + message: + "Native project recovery is incomplete or its ownership changed; selected state is retained. Values omitted.", + }); + } return startupFailure(error); } finally { process.off("SIGINT", cancel); diff --git a/tests/native-authored-command.test.ts b/tests/native-authored-command.test.ts index e94a03c6e..a86453390 100644 --- a/tests/native-authored-command.test.ts +++ b/tests/native-authored-command.test.ts @@ -2,6 +2,7 @@ import { afterEach, expect, test } from "bun:test"; import { mkdir, mkdtemp, readdir, realpath, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import type { recoverNativeAuthoredProject } from "../src/backends/native-authored-project-recovery.ts"; import { NativeAuthoredProjectStartError, type serveNativeAuthoredProject, @@ -17,6 +18,7 @@ afterEach(async () => { ); }); type Startup = Parameters[0]; +type Recovery = Parameters[0]; async function fixture() { const root = await realpath( await mkdtemp(join(tmpdir(), "native-authored-command-")) @@ -105,6 +107,119 @@ test.each([ }); const macTest = process.platform === "darwin" ? test : test.skip; +macTest( + "explicit down recovery captures saved scope and never delegates to startup", + async () => { + const selected = await fixture(); + const options = { + ...selected.options, + operation: "down" as const, + recover: true, + instance: "feature/$exact", + }; + let calls = 0; + let recovery: Recovery | undefined; + const pending = tryNativeAuthoredCommand({ + ...selected, + options, + serve: () => { + throw new Error("Recovery cannot invoke startup."); + }, + recover: (value) => { + calls++; + recovery = value; + return Promise.reject(new Error("Private diagnostic must not escape.")); + }, + }).catch((error: unknown) => error); + options.instance = "replacement"; + selected.env.HACK_NATIVE_HOME = "/replacement"; + const result = await pending; + expect(calls).toBe(1); + expect(recovery?.scope).toEqual({ + projectRoot: selected.root, + projectDir: join(selected.root, ".hack"), + nativeHome: join(selected.root, "candidate"), + branch: "feature/$exact", + }); + expect(recovery?.timeoutMs).toBe(1500); + expect(result).toHaveProperty("code", "E_LIFECYCLE_FAILED"); + expect(String(result)).not.toContain("Private diagnostic"); + expect(await readdir(join(selected.root, ".hack"))).toEqual([]); + } +); + +macTest( + "recovery refuses env/profile/effect overrides before runtime pin or stored owner selection", + async () => { + for (const extra of [ + { overlay: null }, + { overlay: "qa" }, + { profiles: [] }, + { profiles: ["dev"] }, + { services: [] }, + { json: true }, + { detach: true }, + { unsupportedOptions: true }, + ]) { + const selected = await fixture(); + let calls = 0; + const result = await tryNativeAuthoredCommand({ + ...selected, + options: { + ...selected.options, + operation: "down", + recover: true, + ...extra, + }, + env: { HACK_RUNTIME_BACKEND: "native" }, + recover: () => { + calls++; + throw new Error("Owner must not run."); + }, + }).catch((error: unknown) => error); + expect(result).toHaveProperty("code", "E_NATIVE_PROJECT_UNSUPPORTED"); + expect(calls).toBe(0); + expect(await readdir(join(selected.root, ".hack"))).toEqual([]); + } + } +); + +macTest( + "recovery cancellation awaits the selected owner and restores command signal listeners", + async () => { + const selected = await fixture(); + const before = [ + process.listenerCount("SIGINT"), + process.listenerCount("SIGTERM"), + ]; + const entered = Promise.withResolvers(); + let aborted = false; + const pending = tryNativeAuthoredCommand({ + ...selected, + options: { ...selected.options, operation: "down", recover: true }, + recover: (value) => + new Promise((_, reject) => { + value.signal?.addEventListener( + "abort", + () => { + aborted = true; + reject(new Error("Retained selected recovery.")); + }, + { once: true } + ); + entered.resolve(); + }), + }).catch((error: unknown) => error); + await entered.promise; + process.emit("SIGINT"); + expect(await pending).toHaveProperty("code", "E_LIFECYCLE_FAILED"); + expect(aborted).toBe(true); + expect([ + process.listenerCount("SIGINT"), + process.listenerCount("SIGTERM"), + ]).toEqual(before); + } +); macTest( "foreground dispatch captures exact selection and delegates to the lifetime owner", async () => { diff --git a/tests/native-authored-project-recovery.test.ts b/tests/native-authored-project-recovery.test.ts new file mode 100644 index 000000000..cc0955758 --- /dev/null +++ b/tests/native-authored-project-recovery.test.ts @@ -0,0 +1,429 @@ +import { afterEach, expect, spyOn, test } from "bun:test"; +import { createHash } from "node:crypto"; +import * as files from "node:fs/promises"; +import { + chmod, + copyFile, + lstat, + mkdir, + mkdtemp, + readdir, + realpath, + rename, + rm, + unlink, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { parseNativeAuthoredReceipt } from "../src/backends/native-authored-graph-protocol.ts"; +import { recoverNativeAuthoredProject } from "../src/backends/native-authored-project-recovery.ts"; +import { + loadNativeAuthoredProjectRun, + withNativeAuthoredProjectAdmission, +} from "../src/backends/native-authored-project-run.ts"; +import type { invokeNativeRuntime } from "../src/backends/native-runtime-client.ts"; + +const fixtures: Array<{ root: string; child: ReturnType }> = + []; +afterEach(async () => { + for (const current of fixtures.splice(0)) { + if (current.child.exitCode === null) { + current.child.kill("SIGKILL"); + } + await current.child.exited; + await rm(current.root, { recursive: true, force: true }); + } +}); +function receipt() { + const run = "a".repeat(32); + const provenance = { + version: 1, + kind: "native", + namespace: "b".repeat(64), + run, + input: { + semantic_hash: "c".repeat(64), + local_resolution_hash: "d".repeat(64), + environment_policy_hash: "e".repeat(64), + selected_profiles: [], + }, + }; + return parseNativeAuthoredReceipt({ + version: 2, + kind: "native-graph-runtime", + owner: "f".repeat(32), + boot: "00000000-0000-0000-0000-000000000001", + review: { + provenance, + review_id: createHash("sha256") + .update("hack.native-graph-review/v1\0") + .update(JSON.stringify(provenance)) + .digest("hex"), + }, + phase: "ready-observed", + readiness: { web: "healthy" }, + resources: { + "network:default": { + kind: "network", + key: "default", + name: `hkn-${run}-network-0`, + id: "1".repeat(64), + image: null, + phase: "created", + outbound: true, + }, + "container:web": { + kind: "container", + key: "web", + name: `hkn-${run}-container-0`, + id: "2".repeat(64), + image: `sha256:${"3".repeat(64)}`, + phase: "started", + networks: ["default"], + }, + }, + }); +} +function removed() { + const original = receipt(); + return parseNativeAuthoredReceipt({ + ...original, + phase: "removed", + resources: Object.fromEntries( + Object.entries(original.resources).map(([key, value]) => [ + key, + { ...value, phase: "removed" }, + ]) + ), + }); +} +async function fixture() { + const root = await realpath( + await mkdtemp(join(tmpdir(), "native-frontend-recovery-")) + ); + await chmod(root, 0o700); + const scope = { + projectRoot: root, + projectDir: join(root, ".hack"), + nativeHome: join(root, "candidate"), + branch: "qa", + }; + await mkdir(scope.projectDir, { mode: 0o700 }); + await mkdir(scope.nativeHome, { mode: 0o700 }); + const program = [ + `import { withNativeAuthoredProjectAdmission } from ${JSON.stringify(join(import.meta.dir, "../src/backends/native-authored-project-run.ts"))};`, + `const scope = ${JSON.stringify(scope)}; const receipt = ${JSON.stringify(receipt())};`, + "await withNativeAuthoredProjectAdmission(scope, async admission => {", + "await admission.prepareSource({run:receipt.review.provenance.run,metadata:{metadata_version:1,overlay:null,overlay_exists:true,workloads:{web:{TOKEN:{scope:'global',secret:true}}},inactive_scopes:[]}});", + "const start = await admission.reserve({review:receipt.review});", + "await admission.publish({expectedStart:start,record:{version:2,kind:'native-authored-project-run',receipt}});", + "await Bun.write(scope.projectRoot+'/witness','ready');", + "await new Promise(resolve=>setTimeout(resolve,120000)); });", + ].join("\n"); + const child = Bun.spawn([process.execPath, "--eval", program], { + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + env: { PATH: process.env.PATH, TMPDIR: process.env.TMPDIR }, + }); + fixtures.push({ root, child }); + const deadline = performance.now() + 5000; + while (!(await Bun.file(join(root, "witness")).exists())) { + if (child.exitCode !== null || performance.now() >= deadline) { + throw new Error("Owned frontend did not publish Ready."); + } + await Bun.sleep(10); + } + const dir = join(scope.projectDir, ".internal", "native-authored-runs"); + const key = createHash("sha256") + .update(JSON.stringify(scope.branch)) + .digest("hex"); + const paths = { + dir, + ready: join(dir, `${key}.json`), + start: join(dir, `${key}.start.json`), + source: join(dir, `${"a".repeat(32)}.source.json`), + admission: join(dir, `${key}.admission.lock`), + recovery: join(dir, `${key}.admission.recovery`), + intent: join(dir, `${key}.json.recovery.json`), + }; + const calls: string[][] = []; + let cleanupCalls = 0; + const selection = { + version: 1, + kind: "native-graph-recovery-selection", + run: "a".repeat(32), + receipt: receipt(), + receipt_sha256: "4".repeat(64), + owner_sha256: "5".repeat(64), + host_boot_micros: 1_791_000_000_000_000, + }; + const result = { + version: 1, + kind: "native-graph-live-owner-recovered", + run: selection.run, + same_boot: true, + publication_retired: true, + receipt: removed(), + }; + const request: typeof invokeNativeRuntime = (opts) => { + expect(opts.privateInput).toBeUndefined(); + expect(opts.cwd).toBe(root); + calls.push([...opts.args]); + if (opts.args[2] === "recovery-selection") { + expect(opts.args).toEqual([ + "graph", + "native", + "recovery-selection", + "--run-id", + selection.run, + "--json", + ]); + expect(opts.boundNativeAuthoredRecoveryDrain).toBe("selection"); + return Promise.resolve(structuredClone(selection)); + } + if (opts.args[2] === "recover-live-owner") { + expect(opts.args).toEqual([ + "graph", + "native", + "recover-live-owner", + "--run-id", + selection.run, + "--expect-receipt", + selection.receipt_sha256, + "--expect-owner", + selection.owner_sha256, + "--json", + ]); + expect(opts.boundNativeAuthoredRecoveryDrain).toBe("cleanup"); + cleanupCalls++; + return Promise.resolve(structuredClone(result)); + } + expect(opts.args).toEqual([ + "graph", + "native", + "inspect", + "--run-id", + selection.run, + "--json", + ]); + expect(opts.boundNativeAuthoredReadDrain).toBe(true); + return Promise.resolve({ receipt: removed(), observations: { web: null } }); + }; + const options = { + scope, + runtime: { + binary: join(root, "native-request-seam"), + home: scope.nativeHome, + }, + timeoutMs: 30_000, + request, + }; + return { + root, + child, + scope, + paths, + calls, + request, + options, + selection, + result, + cleanupCalls: () => cleanupCalls, + async kill() { + child.kill("SIGKILL"); + expect(await child.exited).toBe(137); + }, + }; +} + +test("explicit frontend recovery refuses a live admission before any native request", async () => { + const current = await fixture(); + await expect(recoverNativeAuthoredProject(current.options)).rejects.toThrow(); + expect(current.calls).toEqual([]); + expect( + (await loadNativeAuthoredProjectRun(current.scope))?.record.receipt + ).toEqual(receipt()); + expect(await Bun.file(current.paths.intent).exists()).toBe(false); +}); + +test("dead frontend recovery binds the stored run, holds startup, retires only after Removed/null and archives completed history", async () => { + const current = await fixture(); + await current.kill(); + let ordinaryAction = false; + const request: typeof invokeNativeRuntime = async (opts) => { + await expect( + withNativeAuthoredProjectAdmission(current.scope, async () => { + ordinaryAction = true; + }) + ).rejects.toThrow(); + return await current.request(opts); + }; + const result = await recoverNativeAuthoredProject({ + ...current.options, + request, + }); + expect(result.receipt.phase).toBe("removed"); + expect(ordinaryAction).toBe(false); + expect(current.cleanupCalls()).toBe(1); + expect((await readdir(current.paths.dir)).sort()).toEqual( + [".gitignore", basename(current.paths.intent)].sort() + ); + const saved = JSON.parse(await Bun.file(current.paths.intent).text()); + expect(saved.record.phase).toBe("complete"); + expect(saved.record.lease_releasing).toBe(true); + await recoverNativeAuthoredProject(current.options); + expect(current.cleanupCalls()).toBe(1); + await withNativeAuthoredProjectAdmission(current.scope, async () => { + ordinaryAction = true; + }); + expect(ordinaryAction).toBe(true); + expect(await readdir(current.paths.dir)).toEqual([ + ".gitignore", + `${current.selection.run}.frontend-recovered.json`, + ]); + expect( + ( + await lstat( + join( + current.paths.dir, + `${current.selection.run}.frontend-recovered.json` + ) + ) + ).nlink + ).toBe(1); +}); + +test("recovery refuses foreign Removed membership or nonnull current observations and retains frontend authority", async () => { + for (const attack of ["id", "observation"]) { + const current = await fixture(); + await current.kill(); + const request: typeof invokeNativeRuntime = async (opts) => { + const value = await current.request(opts); + if (opts.args[2] === "inspect") { + if (attack === "id") { + const receipt = removed(); + return { + receipt: { + ...receipt, + resources: { + ...receipt.resources, + "container:web": { + ...receipt.resources["container:web"], + id: "9".repeat(64), + }, + }, + }, + observations: { web: null }, + }; + } + return { + receipt: removed(), + observations: { web: { state: "running", health: "healthy" } }, + }; + } + return value; + }; + await expect( + recoverNativeAuthoredProject({ ...current.options, request }) + ).rejects.toThrow(); + expect(current.cleanupCalls()).toBe(1); + for (const path of [ + current.paths.ready, + current.paths.start, + current.paths.source, + current.paths.admission, + ]) { + expect(await lstat(path)).toBeDefined(); + } + expect( + JSON.parse(await Bun.file(current.paths.intent).text()).record.phase + ).toBe("native-removed"); + } +}); + +test("same-byte source replacement after cleanup cannot authorize frontend retirement", async () => { + const current = await fixture(); + await current.kill(); + const original = (await lstat(current.paths.source)).ino; + const request: typeof invokeNativeRuntime = async (opts) => { + const value = await current.request(opts); + if (opts.args[2] === "recover-live-owner") { + await copyFile(current.paths.source, join(current.root, "replacement")); + await chmod(join(current.root, "replacement"), 0o600); + await rename(join(current.root, "replacement"), current.paths.source); + } + return value; + }; + await expect( + recoverNativeAuthoredProject({ ...current.options, request }) + ).rejects.toThrow(); + expect((await lstat(current.paths.source)).ino).not.toBe(original); + expect(await Bun.file(current.paths.ready).exists()).toBe(true); + expect( + JSON.parse(await Bun.file(current.paths.intent).text()).record.phase + ).toBe("prepared"); +}); + +test("retirement intent authorizes only its exact file absence after interrupted unlink", async () => { + const current = await fixture(); + await current.kill(); + const originalUnlink = files.unlink; + let interrupted = false; + const spy = spyOn(files, "unlink").mockImplementation(async (path) => { + await originalUnlink(path); + if (path === current.paths.ready && !interrupted) { + interrupted = true; + throw new Error("Exact unlink completed before interrupted phase save."); + } + }); + try { + await expect( + recoverNativeAuthoredProject(current.options) + ).rejects.toThrow(); + } finally { + spy.mockRestore(); + } + expect(interrupted).toBe(true); + expect(await Bun.file(current.paths.ready).exists()).toBe(false); + expect( + JSON.parse(await Bun.file(current.paths.intent).text()).record.phase + ).toBe("ready-retirement"); + await recoverNativeAuthoredProject(current.options); + expect(current.cleanupCalls()).toBe(1); + expect( + JSON.parse(await Bun.file(current.paths.intent).text()).record.phase + ).toBe("complete"); +}); + +test("unexpected ready absence and pending intent never become retry or ordinary startup authority", async () => { + const current = await fixture(); + await current.kill(); + const request: typeof invokeNativeRuntime = async (opts) => { + const value = await current.request(opts); + if (opts.args[2] === "inspect") { + throw new Error("Inspection retained."); + } + return value; + }; + await expect( + recoverNativeAuthoredProject({ ...current.options, request }) + ).rejects.toThrow(); + expect( + JSON.parse(await Bun.file(current.paths.intent).text()).record.phase + ).toBe("native-removed"); + await unlink(current.paths.ready); + const before = current.calls.length; + await expect(recoverNativeAuthoredProject(current.options)).rejects.toThrow(); + expect(current.calls.length).toBe(before); + await Bun.write(`${current.paths.intent}.pending`, "retained"); + await chmod(`${current.paths.intent}.pending`, 0o600); + await expect(recoverNativeAuthoredProject(current.options)).rejects.toThrow(); + let called = false; + await expect( + withNativeAuthoredProjectAdmission(current.scope, async () => { + called = true; + }) + ).rejects.toThrow(); + expect(called).toBe(false); +}); From 940f63f17cb8f5ed4bf7bba72285d303310a3a8c Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 16:21:07 -0400 Subject: [PATCH 25/47] fix: bind recovery lease candidates before publication --- docs/reference/native-authored-commands.md | 8 + .../native-authored-project-recovery.ts | 325 ++++++++++++------ src/lib/native-compose-private-state.ts | 287 ++++++++++++++++ .../native-authored-project-recovery.test.ts | 176 +++++++++- 4 files changed, 681 insertions(+), 115 deletions(-) diff --git a/docs/reference/native-authored-commands.md b/docs/reference/native-authored-commands.md index 6644a9147..71b378eed 100644 --- a/docs/reference/native-authored-commands.md +++ b/docs/reference/native-authored-commands.md @@ -56,6 +56,14 @@ history is archived under fresh startup admission before a later generation can start; incomplete history never permits automatic takeover. Persistent data stays retained, and no recovery operation reboots the guest or allocates new resources. +Dead recovery lease takeover retains its private directory and commits the exact +next owner file before replacing the old owner. A recorded candidate may be pending +or already published after interruption; both forms must match the saved inode and +bytes before a fresh lease is issued. Separate committed release flags authorize +only the corresponding owner/directory absence. An unrecorded pending candidate +and an interrupted completed-history hardlink archive remain retained refusals; +this operation does not repair arbitrary partial lock or file publications. + The native receipt and source paths are distinct from strict Compose v1 artifacts. No native hash substitutes for a normalized Compose hash. Source and fake-driver checks do not qualify an installed frontend, a live provider, the full authored diff --git a/src/backends/native-authored-project-recovery.ts b/src/backends/native-authored-project-recovery.ts index 54688bd08..d132da1bb 100644 --- a/src/backends/native-authored-project-recovery.ts +++ b/src/backends/native-authored-project-recovery.ts @@ -66,7 +66,9 @@ type Intent = { readonly lease: NativeComposeInterruptedLockSelection; readonly mutation: NativeComposeInterruptedLockSelection; readonly lease_releasing: boolean; - readonly mutation_retirement: boolean; + readonly mutation_releasing: boolean; + readonly next_lease: NativeComposeInterruptedLockSelection | null; + readonly next_mutation: NativeComposeInterruptedLockSelection | null; }; type Store = Parameters< Parameters[1] @@ -111,7 +113,7 @@ function parseIntent(value: unknown): Intent { isRecord(value) && keys( value, - "admission,kind,lease,lease_releasing,mutation,mutation_retirement,native,phase,ready,source,start,version" + "admission,kind,lease,lease_releasing,mutation,mutation_releasing,native,next_lease,next_mutation,phase,ready,source,start,version" ) ) || value.version !== 1 || @@ -119,7 +121,7 @@ function parseIntent(value: unknown): Intent { typeof value.phase !== "string" || !PHASES.includes(value.phase as Phase) || typeof value.lease_releasing !== "boolean" || - typeof value.mutation_retirement !== "boolean" + typeof value.mutation_releasing !== "boolean" ) { return refused(); } @@ -128,11 +130,22 @@ function parseIntent(value: unknown): Intent { const admission = parseNativeComposeInterruptedLockSelection(value.admission); const lease = parseNativeComposeInterruptedLockSelection(value.lease); const mutation = parseNativeComposeInterruptedLockSelection(value.mutation); + const nextLease = + value.next_lease === null + ? null + : parseNativeComposeInterruptedLockSelection(value.next_lease); + const nextMutation = + value.next_mutation === null + ? null + : parseNativeComposeInterruptedLockSelection(value.next_mutation); if ( JSON.stringify(start.record.review) !== JSON.stringify(ready.record.receipt.review) || lease.owner.bootId !== admission.owner.bootId || - mutation.owner.bootId !== admission.owner.bootId + mutation.owner.bootId !== admission.owner.bootId || + (nextLease !== null && nextLease.owner.bootId !== admission.owner.bootId) || + (nextMutation !== null && + nextMutation.owner.bootId !== admission.owner.bootId) ) { return refused(); } @@ -151,7 +164,9 @@ function parseIntent(value: unknown): Intent { lease, mutation, lease_releasing: value.lease_releasing, - mutation_retirement: value.mutation_retirement, + mutation_releasing: value.mutation_releasing, + next_lease: nextLease, + next_mutation: nextMutation, }); } function rank(record: Intent): number { @@ -253,6 +268,68 @@ async function readIntent(store: Store): Promise { }, }; } +type PublicationGuard = Pick< + NativeComposeMutationLease, + "assertHeld" | "assertActive" +>; +async function publishIntent(opts: { + readonly store: Store; + readonly previous: Saved | null; + readonly record: Intent; + readonly guard: PublicationGuard; + readonly other?: PublicationGuard; +}): Promise { + const { store, previous, guard, other } = opts; + const record = parseIntent(opts.record); + const text = JSON.stringify({ scope: store.scope, record }); + if (Buffer.byteLength(text) > LIMIT) { + return refused(); + } + await guard.assertHeld(); + await other?.assertHeld(); + await store.check(); + if (JSON.stringify(await readIntent(store)) !== JSON.stringify(previous)) { + return refused(); + } + const info = await writeExclusive(pendingPath(store), text); + await guard.assertHeld(); + await other?.assertHeld(); + await store.check(); + const pending = await readPrivate(pendingPath(store), LIMIT); + if (!sameFile(pending.info, info) || pending.text !== text) { + return refused(); + } + if (previous) { + const old = await readPrivate(intentPath(store), LIMIT); + if ( + !sameFile(old.info, previous.identity) || + hash(old.text) !== previous.identity.sha256 + ) { + return refused(); + } + guard.assertActive(); + other?.assertActive(); + await rename(pendingPath(store), intentPath(store)); + } else { + guard.assertActive(); + other?.assertActive(); + await link(pendingPath(store), intentPath(store)); + await linkedArchive(store, pendingPath(store), { + dev: info.dev, + ino: info.ino, + sha256: hash(text), + }); + guard.assertActive(); + other?.assertActive(); + await unlink(pendingPath(store)); + } + await synchronizeDirectories(store.held); + const saved = await readIntent(store); + if (!saved || JSON.stringify(saved.record) !== JSON.stringify(record)) { + return refused(); + } + return saved; +} function lock(store: Store, path: string, recoveryPath: string) { return createNativeComposePrivateMutationLock({ lockPath: path, @@ -420,24 +497,26 @@ export async function recoverNativeAuthoredProject(opts: { opts.scope, async (store) => { let saved = await readIntent(store); + const record = () => saved?.record ?? refused(); + const update = async ( + value: Intent, + guard: PublicationGuard, + other?: PublicationGuard + ) => { + saved = await publishIntent({ + store, + previous: saved, + record: value, + guard, + other, + }); + }; const recovery = lock( store, store.recovery, `${store.recovery}.recovery` ); - if (!(await absent(store.recovery))) { - if (!saved) { - return refused(); - } - const current = await recovery.selectInterruptedLock(); - if (JSON.stringify(current) !== JSON.stringify(saved.record.lease)) { - return refused(); - } - await recovery.recoverSelectedInterruptedLock(current); - } else if (saved && !saved.record.lease_releasing) { - return refused(); - } - return await recovery.withLock(async (lease) => { + const primaryRun = async (lease: NativeComposeMutationLease) => { const check = async () => { await lease.assertHeld(); await store.check(); @@ -446,101 +525,22 @@ export async function recoverNativeAuthoredProject(opts: { return refused(); } }; - const commit = async ( - value: Intent, - mutation?: NativeComposeMutationLease - ) => { - const active = () => { - lease.assertActive(); - mutation?.assertActive(); - }; - const record = parseIntent(value); - const text = JSON.stringify({ scope: store.scope, record }); - if (Buffer.byteLength(text) > LIMIT) { - return refused(); - } - await check(); - await mutation?.assertHeld(); - const info = await writeExclusive(pendingPath(store), text); - // This attempt's own pending file is allowed only at this publication boundary. - await lease.assertHeld(); - await mutation?.assertHeld(); - await store.check(); - const current = await readPrivate(pendingPath(store), LIMIT); - if (!sameFile(current.info, info) || current.text !== text) { - return refused(); - } - if (saved) { - const old = await readPrivate(intentPath(store), LIMIT); - if ( - !sameFile(old.info, saved.identity) || - hash(old.text) !== saved.identity.sha256 - ) { - return refused(); - } - active(); - await rename(pendingPath(store), intentPath(store)); - } else { - active(); - await link(pendingPath(store), intentPath(store)); - active(); - await unlink(pendingPath(store)); - } - await synchronizeDirectories(store.held); - saved = await readIntent(store); - if ( - !saved || - JSON.stringify(saved.record) !== JSON.stringify(record) - ) { - return refused(); - } - }; + const commit = (value: Intent, mutation?: NativeComposeMutationLease) => + update(value, lease, mutation); const mutation = lock( store, store.mutation, `${store.mutation}.recovery` ); - if (!(await absent(store.mutation))) { - if (!saved) { - return refused(); - } - if (!saved.record.mutation_retirement) { - const selected = await mutation.selectInterruptedLock(); - if ( - JSON.stringify(selected) !== JSON.stringify(saved.record.mutation) - ) { - return refused(); - } - } - await commit({ - ...saved.record, - lease: lease.selection, - mutation_retirement: true, - }); - await lock( - store, - store.mutation, - store.recovery - ).retireSelectedUnderRecoveryLease({ - selected: saved.record.mutation, - lease, - allowOwnerAbsent: true, - }); - } else if ( - saved && - !saved.record.lease_releasing && - !saved.record.mutation_retirement - ) { - return refused(); - } - return await mutation.withLock(async (mutationLease) => { - const record = () => saved?.record ?? refused(); + const mutationRun = async ( + mutationLease: NativeComposeMutationLease + ) => { const both = async () => { remaining(); await check(); await mutationLease.assertHeld(); }; - try { + { if (saved) { await commit( { @@ -548,7 +548,9 @@ export async function recoverNativeAuthoredProject(opts: { lease: lease.selection, mutation: mutationLease.selection, lease_releasing: false, - mutation_retirement: false, + mutation_releasing: false, + next_lease: null, + next_mutation: null, }, mutationLease ); @@ -595,7 +597,9 @@ export async function recoverNativeAuthoredProject(opts: { lease: lease.selection, mutation: mutationLease.selection, lease_releasing: false, - mutation_retirement: false, + mutation_releasing: false, + next_lease: null, + next_mutation: null, }, mutationLease ); @@ -697,16 +701,105 @@ export async function recoverNativeAuthoredProject(opts: { publication_retired: true, receipt: removed, }; - } finally { - if (saved) { - await commit( - { ...saved.record, lease_releasing: true }, - mutationLease - ); + } + }; + const releaseMutation = async ( + mutationLease: NativeComposeMutationLease + ) => { + if (saved) { + await commit( + { + ...record(), + mutation: mutationLease.selection, + next_mutation: null, + mutation_releasing: true, + }, + mutationLease + ); + } + }; + if (!saved) { + return await mutation.withLock(async (mutationLease) => { + try { + return await mutationRun(mutationLease); + } finally { + await releaseMutation(mutationLease); } + }); + } + return await mutation.withPreparedRecoveryLock( + { + expected: record().mutation, + successor: record().next_mutation ?? undefined, + allowOwnerAbsent: record().mutation_releasing, + reserve: (reservation) => + update( + { + ...record(), + mutation: reservation.previous, + next_mutation: reservation.next, + mutation_releasing: reservation.previousAbsent, + }, + reservation, + lease + ), + release: releaseMutation, + }, + mutationRun + ); + }; + const releasePrimary = async (lease: NativeComposeMutationLease) => { + if (saved) { + await update( + { + ...record(), + lease: lease.selection, + next_lease: null, + lease_releasing: true, + }, + lease + ); + } + }; + if (!saved) { + return await recovery.withLock(async (lease) => { + try { + return await primaryRun(lease); + } finally { + await releasePrimary(lease); } }); - }); + } + return await recovery.withPreparedRecoveryLock( + { + expected: record().lease, + successor: record().next_lease ?? undefined, + allowOwnerAbsent: record().lease_releasing, + reserve: (reservation) => + update( + { + ...record(), + lease: reservation.previous, + next_lease: reservation.next, + lease_releasing: reservation.previousAbsent, + }, + reservation + ), + release: releasePrimary, + }, + async (lease) => { + await update( + { + ...record(), + lease: lease.selection, + next_lease: null, + lease_releasing: false, + }, + lease + ); + return await primaryRun(lease); + } + ); } ); } @@ -723,7 +816,13 @@ export async function archiveCompletedNativeAuthoredRecovery(opts: { if (!saved) { return; } - if (saved.record.phase !== "complete" || !saved.record.lease_releasing) { + if ( + saved.record.phase !== "complete" || + !saved.record.lease_releasing || + !saved.record.mutation_releasing || + saved.record.next_lease !== null || + saved.record.next_mutation !== null + ) { return refused(); } const mutation = lock(store, store.mutation, `${store.mutation}.recovery`); diff --git a/src/lib/native-compose-private-state.ts b/src/lib/native-compose-private-state.ts index b915b65c7..3f0773eac 100644 --- a/src/lib/native-compose-private-state.ts +++ b/src/lib/native-compose-private-state.ts @@ -7,6 +7,7 @@ import { open, readdir, realpath, + rename, rmdir, unlink, } from "node:fs/promises"; @@ -59,6 +60,15 @@ export type NativeComposeInterruptedLockSelection = { }; readonly owner: LockOwner; }; +/** A bounded candidate publication, not an issued mutation lease. The callback + * must durably bind both selectors before this owner promotes the candidate. */ +export type NativeComposePreparedLockReservation = { + readonly previous: NativeComposeInterruptedLockSelection; + readonly previousAbsent: boolean; + readonly next: NativeComposeInterruptedLockSelection; + readonly assertHeld: () => Promise; + readonly assertActive: () => undefined; +}; function fileIdentity(value: unknown): value is Record & { readonly dev: number; @@ -653,6 +663,280 @@ export function createNativeComposePrivateMutationLock(opts: { } }; + const withPreparedRecoveryLock = async ( + input: { + readonly expected: unknown; + readonly successor?: unknown; + /** Only a pre-existing committed release phase may permit absence. */ + readonly allowOwnerAbsent: boolean; + readonly reserve: ( + reservation: NativeComposePreparedLockReservation + ) => Promise; + readonly release: (lease: NativeComposeMutationLease) => Promise; + }, + run: (lease: NativeComposeMutationLease) => Promise + ): Promise => { + const expected = parseNativeComposeInterruptedLockSelection(input.expected); + const successor = + input.successor === undefined + ? undefined + : parseNativeComposeInterruptedLockSelection(input.successor); + const allowOwnerAbsent = input.allowOwnerAbsent; + const reserve = input.reserve; + const release = input.release; + if (typeof allowOwnerAbsent !== "boolean") { + return refuse(); + } + await check(); + await requireAbsentGuard(recoveryPath); + if ((await bootId()) !== expected.owner.bootId) { + return refuse(); + } + const prepareDirectory = async () => { + try { + await lstat(lockPath); + return true; + } catch (error) { + if ( + !(hasCode(error, "ENOENT") && allowOwnerAbsent) || + successor !== undefined + ) { + throw error; + } + await mkdir(lockPath, { mode: 0o700 }); + return false; + } + }; + const directoryExists = await prepareDirectory(); + const lock = await holdDirectory(lockPath, true); + const ownerPath = join(lockPath, "owner"); + const pendingPath = join(lockPath, "owner.pending"); + let active = true; + let reservationActive = false; + let issued: NativeComposeMutationLease | undefined; + const checkDirectory = async () => { + if (!active) { + return refuse(); + } + await check(); + await requireAbsentGuard(recoveryPath); + await recheckDirectories([lock]); + if (!active) { + return refuse(); + } + }; + const readOwner = async () => { + try { + return interruptedSelection( + lock, + await readPrivate(ownerPath, LOCK_OWNER_LIMIT) + ); + } catch (error) { + if (hasCode(error, "ENOENT")) { + return undefined; + } + throw error; + } + }; + const sameSelection = ( + left: NativeComposeInterruptedLockSelection, + right: NativeComposeInterruptedLockSelection + ) => JSON.stringify(left) === JSON.stringify(right); + const matchesAdmitted = (value: NativeComposeInterruptedLockSelection) => + sameSelection(value, expected) || + (successor !== undefined && sameSelection(value, successor)); + const promoteSavedCandidate = async ( + current: NativeComposeInterruptedLockSelection | undefined + ) => { + if ( + !successor || + (current !== undefined && !sameSelection(current, expected)) + ) { + return refuse(); + } + const pending = interruptedSelection( + lock, + await readPrivate(pendingPath, LOCK_OWNER_LIMIT) + ); + requireInterruptedSelection(successor, pending); + await requireDeadOwner(successor.owner); + await checkDirectory(); + if (JSON.stringify(await readOwner()) !== JSON.stringify(current)) { + return refuse(); + } + requireInterruptedSelection( + successor, + interruptedSelection( + lock, + await readPrivate(pendingPath, LOCK_OWNER_LIMIT) + ) + ); + if (!active) { + return refuse(); + } + await rename(pendingPath, ownerPath); + await lock.file.sync(); + const promoted = await readOwner(); + if (!(promoted && sameSelection(promoted, successor))) { + return refuse(); + } + return promoted; + }; + const admitDirectory = async () => { + if ( + directoryExists && + !sameFile(lock.info, expected.directory) && + !(successor && sameFile(lock.info, successor.directory)) + ) { + return refuse(); + } + const names = (await readdir(lockPath)).sort(); + if (names.some((name) => name !== "owner" && name !== "owner.pending")) { + return refuse(); + } + return names; + }; + const admitOwner = async () => { + const names = await admitDirectory(); + const current = await readOwner(); + if (current && !matchesAdmitted(current)) { + return refuse(); + } + if (!(current || allowOwnerAbsent)) { + return refuse(); + } + if (current) { + await requireDeadOwner(current.owner); + } + if (names.includes("owner.pending")) { + return await promoteSavedCandidate(current); + } + if (successor && !(current && sameSelection(current, successor))) { + // A reserved candidate may neither disappear nor become an empty dir. + return refuse(); + } + return current; + }; + const runIssuedLease = async (lease: NativeComposeMutationLease) => { + const result = await run(lease).then( + (value) => ({ ok: true as const, value }), + (error: unknown) => ({ ok: false as const, error }) + ); + // Failed release publication retains this exact owner and directory. + // Issuance spans the awaited release callback, then ends synchronously. + try { + await release(lease); + } finally { + mutationLeases.delete(lease); + } + await lease.assertHeld(); + if (!active) { + return refuse(); + } + await unlink(ownerPath); + await checkDirectory(); + if ((await readdir(lockPath)).length !== 0 || !active) { + return refuse(); + } + await rmdir(lockPath); + await parent?.file.sync(); + if (result.ok) { + return result.value; + } + throw result.error; + }; + try { + const current = await admitOwner(); + const previous = current ?? expected; + await checkDirectory(); + if (JSON.stringify(await readOwner()) !== JSON.stringify(current)) { + return refuse(); + } + const owner = await captureLockOwner(); + const text = JSON.stringify(owner); + const info = await writeExclusive(pendingPath, text); + await lock.file.sync(); + const next = interruptedSelection(lock, { info, text }); + reservationActive = true; + const assertReserved = async () => { + if (!reservationActive) { + return refuse(); + } + await checkDirectory(); + if (JSON.stringify(await readOwner()) !== JSON.stringify(current)) { + return refuse(); + } + const pending = await readPrivate(pendingPath, LOCK_OWNER_LIMIT); + if ( + !sameFile(pending.info, info) || + pending.text !== text || + !reservationActive + ) { + return refuse(); + } + }; + await reserve( + Object.freeze({ + previous, + previousAbsent: current === undefined, + next, + assertHeld: assertReserved, + assertActive: () => { + if (!(active && reservationActive)) { + return refuse(); + } + return undefined; + }, + }) + ); + await assertReserved(); + if (current) { + await requireDeadOwner(current.owner); + } + await assertReserved(); + if (!(active && reservationActive)) { + return refuse(); + } + await rename(pendingPath, ownerPath); + reservationActive = false; + await lock.file.sync(); + const assertHeld = async () => { + await checkDirectory(); + if (JSON.stringify(await readdir(lockPath)) !== '["owner"]') { + return refuse(); + } + const latest = await readPrivate(ownerPath, LOCK_OWNER_LIMIT); + if (!sameFile(latest.info, info) || latest.text !== text || !active) { + return refuse(); + } + }; + await assertHeld(); + const lease: NativeComposeMutationLease = Object.freeze({ + token: owner.token, + directory: Object.freeze({ dev: lock.info.dev, ino: lock.info.ino }), + owner: Object.freeze({ dev: info.dev, ino: info.ino }), + selection: next, + assertHeld, + assertActive: () => { + if (!active || issued === undefined || !mutationLeases.has(issued)) { + return refuse(); + } + return undefined; + }, + }); + issued = lease; + mutationLeases.add(lease); + return await runIssuedLease(lease); + } finally { + if (issued) { + mutationLeases.delete(issued); + } + reservationActive = false; + active = false; + await lock.file.close(); + } + }; + const retireInterruptedLock = async ( recovery: HeldDirectory, expected: NativeComposeInterruptedLockSelection | undefined @@ -741,6 +1025,9 @@ export function createNativeComposePrivateMutationLock(opts: { return { withLock, + /** Stable-directory takeover for an exact durable successor/release protocol. + * Unknown pending publications refuse; this never issues authority from data. */ + withPreparedRecoveryLock, /** Native startup admission uses the same active lease without material authority. */ withHeldLock: (run: (assertHeld: () => Promise) => Promise) => withLock((lease) => run(lease.assertHeld)), diff --git a/tests/native-authored-project-recovery.test.ts b/tests/native-authored-project-recovery.test.ts index cc0955758..cde3b6e74 100644 --- a/tests/native-authored-project-recovery.test.ts +++ b/tests/native-authored-project-recovery.test.ts @@ -26,12 +26,15 @@ import type { invokeNativeRuntime } from "../src/backends/native-runtime-client. const fixtures: Array<{ root: string; child: ReturnType }> = []; afterEach(async () => { - for (const current of fixtures.splice(0)) { + const retained = fixtures.splice(0); + for (const current of retained) { if (current.child.exitCode === null) { current.child.kill("SIGKILL"); } await current.child.exited; - await rm(current.root, { recursive: true, force: true }); + } + for (const root of new Set(retained.map((item) => item.root))) { + await rm(root, { recursive: true, force: true }); } }); function receipt() { @@ -427,3 +430,172 @@ test("unexpected ready absence and pending intent never become retry or ordinary ).rejects.toThrow(); expect(called).toBe(false); }); + +async function interruptTakeover( + current: Awaited>, + stage: "reserved" | "promoted" | "released-owner" | "unreserved", + kill = true +) { + const witness = join(current.root, `takeover-${stage}`); + const program = [ + "import {spyOn} from 'bun:test'; import * as files from 'node:fs/promises';", + `import {recoverNativeAuthoredProject} from ${JSON.stringify(join(import.meta.dir, "../src/backends/native-authored-project-recovery.ts"))};`, + `const paths=${JSON.stringify(current.paths)}; const stage=${JSON.stringify(stage)}; const witness=${JSON.stringify(witness)};`, + "const pause=async()=>{await Bun.write(witness,stage);await new Promise(resolve=>setTimeout(resolve,120000));};", + "const rename=files.rename, unlink=files.unlink, open=files.open;", + "spyOn(files,'rename').mockImplementation(async(from,to)=>{", + "if(stage==='reserved'&&from===paths.recovery+'/owner.pending'&&to===paths.recovery+'/owner'){await pause();}", + "await rename(from,to);", + "if(stage==='promoted'&&from===paths.recovery+'/owner.pending'&&to===paths.recovery+'/owner'){await pause();}", + "});", + "spyOn(files,'unlink').mockImplementation(async(path)=>{await unlink(path);if(stage==='released-owner'&&path===paths.recovery+'/owner'){await pause();}});", + "spyOn(files,'open').mockImplementation(async(...args)=>{if(stage==='unreserved'&&args[0]===paths.intent+'.pending'){await pause();}return Reflect.apply(open,files,args);});", + `const selection=${JSON.stringify(current.selection)};const result=${JSON.stringify(current.result)};`, + `await recoverNativeAuthoredProject({scope:${JSON.stringify(current.scope)},runtime:${JSON.stringify(current.options.runtime)},timeoutMs:30000,request:async opts=>{`, + "if(opts.args[2]!=='inspect'){throw new Error('Already removed retry must not replay cleanup.');}", + "return {receipt:result.receipt,observations:{web:null}};}});", + "await Bun.write(witness,'unexpected-completion');", + ].join("\n"); + const child = Bun.spawn([process.execPath, "--eval", program], { + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + env: { PATH: process.env.PATH, TMPDIR: process.env.TMPDIR }, + }); + fixtures.push({ root: current.root, child }); + const deadline = performance.now() + 5000; + while (!(await Bun.file(witness).exists())) { + if (child.exitCode !== null || performance.now() >= deadline) { + throw new Error("Owned takeover did not reach the named interruption."); + } + await Bun.sleep(10); + } + expect(await Bun.file(witness).text()).toBe(stage); + if (kill) { + child.kill("SIGKILL"); + expect(await child.exited).toBe(137); + } + return child; +} + +test("durably reserved takeover survives exact abrupt exits before promotion, after promotion and during final release", async () => { + for (const stage of ["reserved", "promoted", "released-owner"] as const) { + const current = await fixture(); + await current.kill(); + const request: typeof invokeNativeRuntime = async (opts) => { + const value = await current.request(opts); + if (opts.args[2] === "inspect") { + throw new Error("Retain native-removed for the next exact attempt."); + } + return value; + }; + await expect( + recoverNativeAuthoredProject({ ...current.options, request }) + ).rejects.toThrow(); + const original = JSON.parse( + await Bun.file(current.paths.intent).text() + ).record; + expect(original.phase).toBe("native-removed"); + await interruptTakeover(current, stage); + const interrupted = JSON.parse( + await Bun.file(current.paths.intent).text() + ).record; + for (const key of ["ready", "start", "source", "admission", "native"]) { + expect(interrupted[key]).toEqual(original[key]); + } + if (stage === "released-owner") { + expect(interrupted.lease_releasing).toBe(true); + expect(interrupted.next_lease).toBeNull(); + expect(await readdir(current.paths.recovery)).toEqual([]); + } else { + expect(interrupted.next_lease).not.toBeNull(); + expect( + ( + await lstat( + join( + current.paths.recovery, + stage === "reserved" ? "owner.pending" : "owner" + ) + ) + ).ino + ).toBe(interrupted.next_lease.file.ino); + } + expect( + (await recoverNativeAuthoredProject(current.options)).receipt.phase + ).toBe("removed"); + expect(current.cleanupCalls()).toBe(1); + expect( + JSON.parse(await Bun.file(current.paths.intent).text()).record.phase + ).toBe("complete"); + } +}, 30_000); + +test("an abrupt exit before candidate reservation retains unknown pending publication without runtime replay", async () => { + const current = await fixture(); + await current.kill(); + const request: typeof invokeNativeRuntime = async (opts) => { + const value = await current.request(opts); + if (opts.args[2] === "inspect") { + throw new Error("Retain native-removed."); + } + return value; + }; + await expect( + recoverNativeAuthoredProject({ ...current.options, request }) + ).rejects.toThrow(); + await interruptTakeover(current, "unreserved"); + const record = JSON.parse(await Bun.file(current.paths.intent).text()).record; + expect(record.next_lease).toBeNull(); + expect(await readdir(current.paths.recovery)).toEqual(["owner.pending"]); + const before = current.calls.length; + await expect(recoverNativeAuthoredProject(current.options)).rejects.toThrow(); + expect(current.calls.length).toBe(before); + expect(await readdir(current.paths.recovery)).toEqual(["owner.pending"]); +}); + +test("reserved takeover refuses live candidates and same-byte replaced pending inodes before requests", async () => { + for (const attack of ["live", "replacement"] as const) { + const current = await fixture(); + await current.kill(); + const request: typeof invokeNativeRuntime = async (opts) => { + const value = await current.request(opts); + if (opts.args[2] === "inspect") { + throw new Error("Retain native-removed."); + } + return value; + }; + await expect( + recoverNativeAuthoredProject({ ...current.options, request }) + ).rejects.toThrow(); + const child = await interruptTakeover( + current, + "reserved", + attack !== "live" + ); + const pending = join(current.paths.recovery, "owner.pending"); + const original = await lstat(pending); + if (attack === "replacement") { + const replacement = join(current.root, "replacement-owner"); + await Bun.write(replacement, await Bun.file(pending).text()); + await chmod(replacement, 0o600); + await rename(replacement, pending); + expect((await lstat(pending)).ino).not.toBe(original.ino); + } + const before = current.calls.length; + await expect( + recoverNativeAuthoredProject(current.options) + ).rejects.toThrow(); + expect(current.calls.length).toBe(before); + expect(await Bun.file(current.paths.ready).exists()).toBe(true); + if (attack === "live") { + expect(child.exitCode).toBeNull(); + expect((await lstat(pending)).ino).toBe(original.ino); + child.kill("SIGKILL"); + expect(await child.exited).toBe(137); + await recoverNativeAuthoredProject(current.options); + expect(current.cleanupCalls()).toBe(1); + } else { + expect((await lstat(pending)).ino).not.toBe(original.ino); + } + } +}, 30_000); From c59453d3ec739d2d4443bc476668b83db355eb5e Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 16:24:30 -0400 Subject: [PATCH 26/47] fix: retain fresh recovery owner on release refusal --- .../native-authored-project-recovery.ts | 22 +++---- src/lib/native-compose-private-state.ts | 26 ++++++-- .../native-authored-project-recovery.test.ts | 65 +++++++++++++++++++ 3 files changed, 94 insertions(+), 19 deletions(-) diff --git a/src/backends/native-authored-project-recovery.ts b/src/backends/native-authored-project-recovery.ts index d132da1bb..62e088907 100644 --- a/src/backends/native-authored-project-recovery.ts +++ b/src/backends/native-authored-project-recovery.ts @@ -719,13 +719,10 @@ export async function recoverNativeAuthoredProject(opts: { } }; if (!saved) { - return await mutation.withLock(async (mutationLease) => { - try { - return await mutationRun(mutationLease); - } finally { - await releaseMutation(mutationLease); - } - }); + return await mutation.withFreshRecoveryLock( + { release: releaseMutation }, + mutationRun + ); } return await mutation.withPreparedRecoveryLock( { @@ -762,13 +759,10 @@ export async function recoverNativeAuthoredProject(opts: { } }; if (!saved) { - return await recovery.withLock(async (lease) => { - try { - return await primaryRun(lease); - } finally { - await releasePrimary(lease); - } - }); + return await recovery.withFreshRecoveryLock( + { release: releasePrimary }, + primaryRun + ); } return await recovery.withPreparedRecoveryLock( { diff --git a/src/lib/native-compose-private-state.ts b/src/lib/native-compose-private-state.ts index 3f0773eac..f38a247f3 100644 --- a/src/lib/native-compose-private-state.ts +++ b/src/lib/native-compose-private-state.ts @@ -560,7 +560,8 @@ export function createNativeComposePrivateMutationLock(opts: { }) { const { lockPath, recoveryPath, parent, check } = opts; const withLock = async ( - run: (lease: NativeComposeMutationLease) => Promise + run: (lease: NativeComposeMutationLease) => Promise, + beforeRetire?: (lease: NativeComposeMutationLease) => Promise ) => { await check(); await requireAbsentGuard(recoveryPath); @@ -620,10 +621,16 @@ export function createNativeComposePrivateMutationLock(opts: { issued = lease; return await run(lease); } finally { - if (issued !== undefined) { - mutationLeases.delete(issued); - } try { + try { + if (issued !== undefined && beforeRetire !== undefined) { + await beforeRetire(issued); + } + } finally { + if (issued !== undefined) { + mutationLeases.delete(issued); + } + } await assertHeld(); await unlink(ownerPath); await rmdir(lockPath); @@ -1024,7 +1031,16 @@ export function createNativeComposePrivateMutationLock(opts: { }; return { - withLock, + withLock: (run: (lease: NativeComposeMutationLease) => Promise) => + withLock(run), + /** Fresh recovery issuance retains its exact owner if durable release fails. + * Existing generation callers keep the original unconditional finalizer. */ + withFreshRecoveryLock: ( + opts: { + readonly release: (lease: NativeComposeMutationLease) => Promise; + }, + run: (lease: NativeComposeMutationLease) => Promise + ) => withLock(run, opts.release), /** Stable-directory takeover for an exact durable successor/release protocol. * Unknown pending publications refuse; this never issues authority from data. */ withPreparedRecoveryLock, diff --git a/tests/native-authored-project-recovery.test.ts b/tests/native-authored-project-recovery.test.ts index cde3b6e74..ecbc12139 100644 --- a/tests/native-authored-project-recovery.test.ts +++ b/tests/native-authored-project-recovery.test.ts @@ -599,3 +599,68 @@ test("reserved takeover refuses live candidates and same-byte replaced pending i } } }, 30_000); + +test("fresh recovery release failure retains each exact owner until a dead-owner retry", async () => { + for (const guard of ["mutation", "lease"] as const) { + const current = await fixture(); + await current.kill(); + const witness = join(current.root, `fresh-release-${guard}`); + const program = [ + "import {spyOn} from 'bun:test'; import * as files from 'node:fs/promises';", + `import {recoverNativeAuthoredProject} from ${JSON.stringify(join(import.meta.dir, "../src/backends/native-authored-project-recovery.ts"))};`, + `const paths=${JSON.stringify(current.paths)}, guard=${JSON.stringify(guard)}, witness=${JSON.stringify(witness)};`, + "const open=files.open;let rejected=false,cleanupCalls=0;", + "spyOn(files,'open').mockImplementation(async(...args)=>{", + "if(!rejected&&args[0]===paths.intent+'.pending'&&await Bun.file(paths.intent).exists()){", + "const record=(await Bun.file(paths.intent).json()).record;", + "if(record.phase==='complete'&&record.mutation_releasing===(guard==='lease')){rejected=true;throw new Error('Fixed release publication refusal.');}", + "}return Reflect.apply(open,files,args);});", + `const selection=${JSON.stringify(current.selection)},result=${JSON.stringify(current.result)};`, + `try{await recoverNativeAuthoredProject({scope:${JSON.stringify(current.scope)},runtime:${JSON.stringify(current.options.runtime)},timeoutMs:30000,request:async opts=>{`, + "if(opts.args[2]==='recovery-selection'){return selection;}if(opts.args[2]==='recover-live-owner'){cleanupCalls++;return result;}", + "return {receipt:result.receipt,observations:{web:null}};}});throw new Error('Unexpected recovery success.');}", + "catch{if(!rejected){throw new Error('Release boundary not observed.');}}", + "await Bun.write(witness,JSON.stringify({rejected,cleanupCalls}));", + ].join("\n"); + const child = Bun.spawn([process.execPath, "--eval", program], { + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + env: { PATH: process.env.PATH, TMPDIR: process.env.TMPDIR }, + }); + fixtures.push({ root: current.root, child }); + const deadline = performance.now() + 5000; + while (!(await Bun.file(witness).exists())) { + if (child.exitCode !== null || performance.now() >= deadline) { + throw new Error("Owned fresh recovery did not reach release refusal."); + } + await Bun.sleep(10); + } + expect(await child.exited).toBe(0); + expect(await Bun.file(witness).json()).toEqual({ + rejected: true, + cleanupCalls: 1, + }); + const record = JSON.parse( + await Bun.file(current.paths.intent).text() + ).record; + expect(record.phase).toBe("complete"); + expect( + record[guard === "lease" ? "lease_releasing" : "mutation_releasing"] + ).toBe(false); + const owner = + guard === "lease" + ? join(current.paths.recovery, "owner") + : join( + current.paths.dir, + `${basename(current.paths.ready, ".json")}.lock`, + "owner" + ); + expect((await lstat(owner)).ino).toBe(record[guard].file.ino); + expect( + (await recoverNativeAuthoredProject(current.options)).receipt.phase + ).toBe("removed"); + expect(current.cleanupCalls()).toBe(0); + expect(await Bun.file(owner).exists()).toBe(false); + } +}, 30_000); From 8ed1fb887c75fe2990d5f215c89496badba01255 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 16:29:05 -0400 Subject: [PATCH 27/47] fix: preserve recovery result literal types --- src/backends/native-authored-project-recovery.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backends/native-authored-project-recovery.ts b/src/backends/native-authored-project-recovery.ts index 62e088907..1b020be7a 100644 --- a/src/backends/native-authored-project-recovery.ts +++ b/src/backends/native-authored-project-recovery.ts @@ -534,7 +534,7 @@ export async function recoverNativeAuthoredProject(opts: { ); const mutationRun = async ( mutationLease: NativeComposeMutationLease - ) => { + ): Promise => { const both = async () => { remaining(); await check(); From 1393d55925c96eafc43a004864cb57813212f76d Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 16:30:56 -0400 Subject: [PATCH 28/47] test: model native frontend recovery lease ordering --- tests/models/tla/README.md | 40 +++++ .../tla/native-frontend-recovery/Recovery.tla | 150 ++++++++++++++++++ .../changed-binding.cfg | 7 + .../fresh-release-retention-reachable.cfg | 7 + .../tla/native-frontend-recovery/negative.cfg | 7 + .../tla/native-frontend-recovery/positive.cfg | 7 + .../promoted-crash-reachable.cfg | 7 + .../release-crash-reachable.cfg | 7 + .../reserved-crash-reachable.cfg | 7 + .../revoked-authority.cfg | 7 + .../uncommitted-release.cfg | 7 + .../unknown-refusal-reachable.cfg | 7 + 12 files changed, 260 insertions(+) create mode 100644 tests/models/tla/native-frontend-recovery/Recovery.tla create mode 100644 tests/models/tla/native-frontend-recovery/changed-binding.cfg create mode 100644 tests/models/tla/native-frontend-recovery/fresh-release-retention-reachable.cfg create mode 100644 tests/models/tla/native-frontend-recovery/negative.cfg create mode 100644 tests/models/tla/native-frontend-recovery/positive.cfg create mode 100644 tests/models/tla/native-frontend-recovery/promoted-crash-reachable.cfg create mode 100644 tests/models/tla/native-frontend-recovery/release-crash-reachable.cfg create mode 100644 tests/models/tla/native-frontend-recovery/reserved-crash-reachable.cfg create mode 100644 tests/models/tla/native-frontend-recovery/revoked-authority.cfg create mode 100644 tests/models/tla/native-frontend-recovery/uncommitted-release.cfg create mode 100644 tests/models/tla/native-frontend-recovery/unknown-refusal-reachable.cfg diff --git a/tests/models/tla/README.md b/tests/models/tla/README.md index f9d8e6694..6fb2fa49e 100644 --- a/tests/models/tla/README.md +++ b/tests/models/tla/README.md @@ -16,6 +16,46 @@ pinned artifact. Each check has a 120-second timeout, 512 MiB Java heap, two wor and bounded output; temporary TLC metadata is removed after success or failure. No credentials or running VM are needed. +## Native foreground recovery lease takeover + +`native-frontend-recovery/Recovery.tla` models the explicit saved-run frontend +recovery lease protocol. Its checker and exact exploration-count registration +are pending initial qualification. It bounds one lease to two new candidates, +one crash, one immutable resource selection and one possible foreign-binding +substitution. Two initial states cover a fresh active recovery at release and a +dead selected owner awaiting takeover. The same protocol serves the primary and +mutation leases; this model checks one lease and assumes its enclosing required +guard is held. It does not establish the ordering of both leases together. + +Reservation, promotion, issuance, current-owner binding, release intent, owner +unlink and directory removal are separate transitions. A pending selector records +the current candidate's exact identity but grants no active lease. Crashes clear +live authority while retaining committed selectors and release phases. Unknown +pending publication refuses, and expected absence needs the preexisting release +intent. Fresh release-publication failure retains its owner. Retry can continue +from a committed candidate before/after promotion or an owner-unlinked directory; +resource progress still requires actual issued authority and unchanged original +membership. Four unsafe controls remove reservation, release, active-authority or +binding checks. Five reachability controls require completion after the named +crashes, unknown-pending refusal and fresh-release retention. + +| Model boundary | Implementation mapping and separate evidence | +| --- | --- | +| `ReserveCandidate` / `BindCandidate` | `createNativeComposePrivateMutationLock.withPreparedRecoveryLock` exclusively writes `owner.pending`, then the frontend `publishIntent` commits its exact inode/raw-byte selector before promotion. Reservation callbacks change only lease metadata. | +| `PromoteCandidate` / `IssueLease` / `BindIssuedLease` | The shared owner rechecks the current/candidate selectors before rename, issues its private active lease only afterward, and the frontend commits the current owner before resource phases can advance. | +| `AdvanceResources` | `recoverNativeAuthoredProject` requires both actual guards, strict original Ready/start/source selections and an authenticated native Removed receipt with null current observations. The three abstract steps are retirement intent, exact Ready unlink and completion; native resource cleanup and other frontend members are omitted. | +| `ArmRelease` / `RemoveOwner` / `RemoveDirectory` | Frontend primary and mutation release flags commit before the shared owner removes the exact owner and directory. `withFreshRecoveryLock` retains both on a failed durable release callback. | +| `Crash` / `RecoverSavedCandidate` / `ReadmitOwner` | The finite child tests SIGKILL the retained child before promotion, after promotion and after owner unlink, then use the same saved intent to retry without replaying cleanup. | +| `RefuseUnknown` / `ReplaceBinding` / `LoseAuthority` | Implementation controls refuse unbound pending publication, live candidate, same-byte/new-inode candidate replacement and revoked issued authority. The model summarizes those identity fields by one version rather than proving real inode/boot/process readers. | + +Each committed durable write is one abstract action; partial writes, pending +intent refusal, fsync durability, hash/entropy/UTF-8 bounds, real filesystem +substitution, OS process identity, deadlines, callback rejection, signal delivery, +socket/guest boot proof and native environment retirement need separate source and +implementation checks. Hardlink archive interruption remains a retained refusal +and is omitted. No fairness, unlimited-crash recovery, provider effects, old-format +compatibility, data deletion or whole NC05 acceptance is claimed. + ## Native storage witness enrollment and read-only resume `native-storage-witness/Witness.tla` checks a proposed storage continuity protocol. diff --git a/tests/models/tla/native-frontend-recovery/Recovery.tla b/tests/models/tla/native-frontend-recovery/Recovery.tla new file mode 100644 index 000000000..3f801a4b9 --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/Recovery.tla @@ -0,0 +1,150 @@ +---------------------------- MODULE Recovery ---------------------------- +EXTENDS Naturals, TLC +CONSTANTS EnforceReservation, EnforceRelease, EnforceAuthority, EnforceBinding +VARIABLES fresh, directory, owner, pending, savedOwner, savedNext, release, + stage, active, alive, attempts, crashed, crashStage, progress, ready, + binding, unsafePromotion, unsafeAbsence, unsafeEffect +publication == <> +process == <> +history == <> +resources == <> +unsafe == <> +vars == <> +Matches == binding = 1 +Issued == active /\ owner = savedOwner /\ savedNext = 0 /\ alive +Init == /\ fresh \in BOOLEAN + /\ directory = TRUE /\ owner = 1 /\ pending = 0 + /\ savedOwner = 1 /\ savedNext = 0 /\ release = FALSE + /\ stage = IF fresh THEN "leased" ELSE "admitted" + /\ active = fresh /\ alive = fresh /\ attempts = 0 + /\ crashed = FALSE /\ crashStage = "none" + /\ progress = IF fresh THEN 3 ELSE 0 /\ ready = ~fresh + /\ binding = 1 /\ unsafePromotion = FALSE + /\ unsafeAbsence = FALSE /\ unsafeEffect = FALSE +ReserveCandidate == + /\ stage = "admitted" /\ directory /\ attempts < 2 /\ Matches + /\ (owner # 0 \/ release) /\ pending = 0 + /\ pending' = attempts + 2 /\ attempts' = attempts + 1 + /\ stage' = "pending" /\ alive' = TRUE /\ active' = FALSE + /\ UNCHANGED <> +BindCandidate == + /\ stage = "pending" /\ alive /\ Matches /\ pending # 0 + /\ savedOwner' = owner /\ savedNext' = pending + /\ release' = (owner = 0) /\ stage' = "reserved" + /\ UNCHANGED <> +PromoteCandidate == + /\ stage \in {"pending", "reserved"} /\ Matches /\ alive + /\ pending # 0 /\ (EnforceReservation => savedNext = pending) + /\ owner' = pending /\ pending' = 0 /\ stage' = "promoted" + /\ unsafePromotion' = unsafePromotion \/ savedNext # pending + /\ UNCHANGED <> +IssueLease == + /\ stage = "promoted" /\ Matches /\ alive /\ owner = savedNext + /\ active' = TRUE /\ stage' = "issued" + /\ UNCHANGED <> +BindIssuedLease == + /\ stage = "issued" /\ active /\ Matches /\ alive + /\ savedOwner' = owner /\ savedNext' = 0 /\ release' = FALSE + /\ stage' = "leased" + /\ UNCHANGED <> +AdvanceResources == + /\ stage = "leased" /\ progress < 3 + /\ (EnforceAuthority => Issued) /\ (EnforceBinding => Matches) + /\ progress' = progress + 1 + /\ ready' = IF progress = 1 THEN FALSE ELSE ready + /\ unsafeEffect' = unsafeEffect \/ ~Issued \/ ~Matches + /\ UNCHANGED <> +ArmRelease == + /\ stage = "leased" /\ Issued /\ Matches + /\ release' = TRUE /\ stage' = "releasing" + /\ UNCHANGED <> +RemoveOwner == + /\ stage \in {"leased", "releasing"} /\ Issued /\ Matches + /\ (EnforceRelease => release) + /\ owner' = 0 /\ active' = FALSE /\ stage' = "empty" + /\ unsafeAbsence' = unsafeAbsence \/ ~release + /\ UNCHANGED <> +RemoveDirectory == + /\ stage = "empty" /\ release /\ owner = 0 /\ pending = 0 /\ Matches + /\ directory' = FALSE /\ stage' = "done" /\ alive' = FALSE + /\ UNCHANGED <> +FailFreshRelease == + /\ fresh /\ stage = "leased" /\ progress = 3 /\ ~release + /\ stage' = "release-failed" /\ active' = FALSE + /\ UNCHANGED <> +LoseAuthority == + /\ stage = "leased" /\ active /\ active' = FALSE + /\ UNCHANGED <> +ReplaceBinding == + /\ binding = 1 /\ binding' = 2 + /\ UNCHANGED <> +Crash == + /\ stage \in {"pending", "reserved", "promoted", "issued", "leased", + "releasing", "empty", "release-failed"} + /\ ~crashed /\ crashed' = TRUE /\ crashStage' = stage + /\ stage' = "reopen" /\ active' = FALSE /\ alive' = FALSE + /\ UNCHANGED <> +RecoverSavedCandidate == + /\ stage = "reopen" /\ ~alive /\ Matches + /\ directory /\ pending # 0 /\ pending = savedNext + /\ (owner = savedOwner \/ (owner = 0 /\ release)) + /\ owner' = pending /\ pending' = 0 /\ stage' = "admitted" + /\ UNCHANGED <> +ReadmitOwner == + /\ stage = "reopen" /\ ~alive /\ Matches /\ pending = 0 + /\ (owner = savedOwner \/ (owner = savedNext /\ owner # 0) + \/ (owner = 0 /\ release)) + /\ directory' = TRUE /\ stage' = "admitted" + /\ UNCHANGED <> +RefuseUnknown == + /\ stage = "reopen" /\ (pending # 0 /\ pending # savedNext) + /\ stage' = "unknown-retained" + /\ UNCHANGED <> +RefuseChanged == + /\ stage = "reopen" /\ ~Matches /\ stage' = "changed-retained" + /\ UNCHANGED <> +Next == ReserveCandidate \/ BindCandidate \/ PromoteCandidate \/ IssueLease + \/ BindIssuedLease \/ AdvanceResources \/ ArmRelease \/ RemoveOwner + \/ RemoveDirectory \/ FailFreshRelease \/ LoseAuthority \/ ReplaceBinding + \/ Crash \/ RecoverSavedCandidate \/ ReadmitOwner \/ RefuseUnknown + \/ RefuseChanged +Spec == Init /\ [][Next]_vars +TypeOK == /\ fresh \in BOOLEAN /\ directory \in BOOLEAN + /\ owner \in 0..3 /\ pending \in 0..3 /\ savedOwner \in 0..3 + /\ savedNext \in 0..3 /\ release \in BOOLEAN + /\ stage \in {"admitted", "pending", "reserved", "promoted", "issued", + "leased", "releasing", "empty", "done", "reopen", + "release-failed", "unknown-retained", "changed-retained"} + /\ active \in BOOLEAN /\ alive \in BOOLEAN /\ attempts \in 0..2 + /\ crashed \in BOOLEAN + /\ crashStage \in {"none", "pending", "reserved", "promoted", "issued", + "leased", "releasing", "empty", "release-failed"} + /\ progress \in 0..3 /\ ready \in BOOLEAN /\ binding \in 1..2 + /\ unsafePromotion \in BOOLEAN /\ unsafeAbsence \in BOOLEAN + /\ unsafeEffect \in BOOLEAN +NoUnreservedPromotion == ~unsafePromotion +NoUncommittedAbsence == ~unsafeAbsence +NoUnissuedEffect == ~unsafeEffect +NeverReservedCrashCompletion == ~(crashStage = "reserved" /\ progress = 3 /\ stage = "done") +NeverPromotedCrashCompletion == ~(crashStage = "promoted" /\ progress = 3 /\ stage = "done") +NeverReleaseCrashCompletion == ~(crashStage = "empty" /\ progress = 3 /\ stage = "done") +NeverUnknownRefusal == stage # "unknown-retained" +NeverFreshReleaseRetention == ~(stage = "release-failed" /\ owner = 1 /\ directory /\ ~release) +============================================================================= diff --git a/tests/models/tla/native-frontend-recovery/changed-binding.cfg b/tests/models/tla/native-frontend-recovery/changed-binding.cfg new file mode 100644 index 000000000..3761b956c --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/changed-binding.cfg @@ -0,0 +1,7 @@ +CONSTANTS EnforceReservation = TRUE + EnforceRelease = TRUE + EnforceAuthority = TRUE + EnforceBinding = FALSE +SPECIFICATION Spec +INVARIANTS TypeOK NoUnreservedPromotion NoUncommittedAbsence NoUnissuedEffect +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/native-frontend-recovery/fresh-release-retention-reachable.cfg b/tests/models/tla/native-frontend-recovery/fresh-release-retention-reachable.cfg new file mode 100644 index 000000000..6e36f86d8 --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/fresh-release-retention-reachable.cfg @@ -0,0 +1,7 @@ +CONSTANTS EnforceReservation = TRUE + EnforceRelease = TRUE + EnforceAuthority = TRUE + EnforceBinding = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK NoUnreservedPromotion NoUncommittedAbsence NoUnissuedEffect NeverFreshReleaseRetention +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/native-frontend-recovery/negative.cfg b/tests/models/tla/native-frontend-recovery/negative.cfg new file mode 100644 index 000000000..2329991eb --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/negative.cfg @@ -0,0 +1,7 @@ +CONSTANTS EnforceReservation = FALSE + EnforceRelease = TRUE + EnforceAuthority = TRUE + EnforceBinding = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK NoUnreservedPromotion NoUncommittedAbsence NoUnissuedEffect +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/native-frontend-recovery/positive.cfg b/tests/models/tla/native-frontend-recovery/positive.cfg new file mode 100644 index 000000000..694c30c1b --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/positive.cfg @@ -0,0 +1,7 @@ +CONSTANTS EnforceReservation = TRUE + EnforceRelease = TRUE + EnforceAuthority = TRUE + EnforceBinding = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK NoUnreservedPromotion NoUncommittedAbsence NoUnissuedEffect +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/native-frontend-recovery/promoted-crash-reachable.cfg b/tests/models/tla/native-frontend-recovery/promoted-crash-reachable.cfg new file mode 100644 index 000000000..edec01152 --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/promoted-crash-reachable.cfg @@ -0,0 +1,7 @@ +CONSTANTS EnforceReservation = TRUE + EnforceRelease = TRUE + EnforceAuthority = TRUE + EnforceBinding = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK NoUnreservedPromotion NoUncommittedAbsence NoUnissuedEffect NeverPromotedCrashCompletion +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/native-frontend-recovery/release-crash-reachable.cfg b/tests/models/tla/native-frontend-recovery/release-crash-reachable.cfg new file mode 100644 index 000000000..cb7bfbc9a --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/release-crash-reachable.cfg @@ -0,0 +1,7 @@ +CONSTANTS EnforceReservation = TRUE + EnforceRelease = TRUE + EnforceAuthority = TRUE + EnforceBinding = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK NoUnreservedPromotion NoUncommittedAbsence NoUnissuedEffect NeverReleaseCrashCompletion +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/native-frontend-recovery/reserved-crash-reachable.cfg b/tests/models/tla/native-frontend-recovery/reserved-crash-reachable.cfg new file mode 100644 index 000000000..6a78f363d --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/reserved-crash-reachable.cfg @@ -0,0 +1,7 @@ +CONSTANTS EnforceReservation = TRUE + EnforceRelease = TRUE + EnforceAuthority = TRUE + EnforceBinding = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK NoUnreservedPromotion NoUncommittedAbsence NoUnissuedEffect NeverReservedCrashCompletion +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/native-frontend-recovery/revoked-authority.cfg b/tests/models/tla/native-frontend-recovery/revoked-authority.cfg new file mode 100644 index 000000000..d15e4ecaf --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/revoked-authority.cfg @@ -0,0 +1,7 @@ +CONSTANTS EnforceReservation = TRUE + EnforceRelease = TRUE + EnforceAuthority = FALSE + EnforceBinding = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK NoUnreservedPromotion NoUncommittedAbsence NoUnissuedEffect +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/native-frontend-recovery/uncommitted-release.cfg b/tests/models/tla/native-frontend-recovery/uncommitted-release.cfg new file mode 100644 index 000000000..6af419111 --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/uncommitted-release.cfg @@ -0,0 +1,7 @@ +CONSTANTS EnforceReservation = TRUE + EnforceRelease = FALSE + EnforceAuthority = TRUE + EnforceBinding = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK NoUnreservedPromotion NoUncommittedAbsence NoUnissuedEffect +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/native-frontend-recovery/unknown-refusal-reachable.cfg b/tests/models/tla/native-frontend-recovery/unknown-refusal-reachable.cfg new file mode 100644 index 000000000..d2c617b2c --- /dev/null +++ b/tests/models/tla/native-frontend-recovery/unknown-refusal-reachable.cfg @@ -0,0 +1,7 @@ +CONSTANTS EnforceReservation = TRUE + EnforceRelease = TRUE + EnforceAuthority = TRUE + EnforceBinding = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK NoUnreservedPromotion NoUncommittedAbsence NoUnissuedEffect NeverUnknownRefusal +CHECK_DEADLOCK FALSE From f9082340c910930ae71fd071cc79a1d061ed8627 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 16:35:05 -0400 Subject: [PATCH 29/47] test: delimit conditional recovery model values --- tests/models/tla/native-frontend-recovery/Recovery.tla | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/models/tla/native-frontend-recovery/Recovery.tla b/tests/models/tla/native-frontend-recovery/Recovery.tla index 3f801a4b9..1c568955a 100644 --- a/tests/models/tla/native-frontend-recovery/Recovery.tla +++ b/tests/models/tla/native-frontend-recovery/Recovery.tla @@ -15,10 +15,10 @@ Issued == active /\ owner = savedOwner /\ savedNext = 0 /\ alive Init == /\ fresh \in BOOLEAN /\ directory = TRUE /\ owner = 1 /\ pending = 0 /\ savedOwner = 1 /\ savedNext = 0 /\ release = FALSE - /\ stage = IF fresh THEN "leased" ELSE "admitted" + /\ stage = (IF fresh THEN "leased" ELSE "admitted") /\ active = fresh /\ alive = fresh /\ attempts = 0 /\ crashed = FALSE /\ crashStage = "none" - /\ progress = IF fresh THEN 3 ELSE 0 /\ ready = ~fresh + /\ progress = (IF fresh THEN 3 ELSE 0) /\ ready = ~fresh /\ binding = 1 /\ unsafePromotion = FALSE /\ unsafeAbsence = FALSE /\ unsafeEffect = FALSE ReserveCandidate == @@ -57,7 +57,7 @@ AdvanceResources == /\ stage = "leased" /\ progress < 3 /\ (EnforceAuthority => Issued) /\ (EnforceBinding => Matches) /\ progress' = progress + 1 - /\ ready' = IF progress = 1 THEN FALSE ELSE ready + /\ ready' = (IF progress = 1 THEN FALSE ELSE ready) /\ unsafeEffect' = unsafeEffect \/ ~Issued \/ ~Matches /\ UNCHANGED <> From 929e817c25358eebafa8562db1b182985e9035b5 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 16:40:09 -0400 Subject: [PATCH 30/47] test: preserve native unsupported dispatch contract --- tests/native-authored-command-dispatch.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/native-authored-command-dispatch.test.ts b/tests/native-authored-command-dispatch.test.ts index e70d64020..a676b2d29 100644 --- a/tests/native-authored-command-dispatch.test.ts +++ b/tests/native-authored-command-dispatch.test.ts @@ -112,8 +112,9 @@ test.each([ const selected = await fixture(); const value = await invoke({ selected, args, backend: "native" }); expect(value.code).toBe(1); + expect(value.stdout + value.stderr).toContain("E_NATIVE_PROJECT_UNSUPPORTED"); expect(value.stdout + value.stderr).toContain( - "whole-project foreground up on macOS" + "whole-project foreground up or explicit stored-generation down --recover on macOS" ); expect(await Bun.file(join(selected.root, "compiler-called")).exists()).toBe( false From d21f97b4d0d4bd8b4e1d86be41a47c3d06f9d06a Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 16:41:03 -0400 Subject: [PATCH 31/47] test: bind recovery model unsafe state assignments --- tests/models/tla/native-frontend-recovery/Recovery.tla | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/models/tla/native-frontend-recovery/Recovery.tla b/tests/models/tla/native-frontend-recovery/Recovery.tla index 1c568955a..2d326e67b 100644 --- a/tests/models/tla/native-frontend-recovery/Recovery.tla +++ b/tests/models/tla/native-frontend-recovery/Recovery.tla @@ -38,7 +38,7 @@ PromoteCandidate == /\ stage \in {"pending", "reserved"} /\ Matches /\ alive /\ pending # 0 /\ (EnforceReservation => savedNext = pending) /\ owner' = pending /\ pending' = 0 /\ stage' = "promoted" - /\ unsafePromotion' = unsafePromotion \/ savedNext # pending + /\ unsafePromotion' = (unsafePromotion \/ savedNext # pending) /\ UNCHANGED <> @@ -58,7 +58,7 @@ AdvanceResources == /\ (EnforceAuthority => Issued) /\ (EnforceBinding => Matches) /\ progress' = progress + 1 /\ ready' = (IF progress = 1 THEN FALSE ELSE ready) - /\ unsafeEffect' = unsafeEffect \/ ~Issued \/ ~Matches + /\ unsafeEffect' = (unsafeEffect \/ ~Issued \/ ~Matches) /\ UNCHANGED <> ArmRelease == @@ -70,7 +70,7 @@ RemoveOwner == /\ stage \in {"leased", "releasing"} /\ Issued /\ Matches /\ (EnforceRelease => release) /\ owner' = 0 /\ active' = FALSE /\ stage' = "empty" - /\ unsafeAbsence' = unsafeAbsence \/ ~release + /\ unsafeAbsence' = (unsafeAbsence \/ ~release) /\ UNCHANGED <> From 67f521d0fa2cd8ff733fd4992bf0320f59aa5ad2 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 16:45:23 -0400 Subject: [PATCH 32/47] test: register bounded frontend recovery model controls --- scripts/lib/tla-runtime-models.ts | 86 +++++++++++++++++++++++++++++++ tests/models/tla/README.md | 4 +- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/scripts/lib/tla-runtime-models.ts b/scripts/lib/tla-runtime-models.ts index dff8f409f..e2deafcea 100644 --- a/scripts/lib/tla-runtime-models.ts +++ b/scripts/lib/tla-runtime-models.ts @@ -40,6 +40,92 @@ const laterIncompleteMissingLockStages = [ "owner", ] as const; const contracts: readonly ModelContract[] = [ + { + name: "native-frontend-recovery", + module: "Recovery", + states: 602, + invariant: "NoUnreservedPromotion", + action: "PromoteCandidate", + fields: [ + "unsafePromotion = TRUE", + 'stage = "promoted"', + "savedNext = 0", + "pending = 0", + ], + additionalControls: [ + { + name: "uncommitted-release", + negative: true, + invariant: "NoUncommittedAbsence", + action: "RemoveOwner", + fields: [ + "unsafeAbsence = TRUE", + 'stage = "empty"', + "release = FALSE", + "owner = 0", + ], + }, + { + name: "revoked-authority", + negative: true, + invariant: "NoUnissuedEffect", + action: "AdvanceResources", + fields: [ + "unsafeEffect = TRUE", + "active = FALSE", + "binding = 1", + "progress = 1", + ], + }, + { + name: "changed-binding", + negative: true, + invariant: "NoUnissuedEffect", + action: "AdvanceResources", + fields: ["unsafeEffect = TRUE", "binding = 2", "progress = 1"], + }, + { + name: "reserved-crash-reachable", + negative: true, + invariant: "NeverReservedCrashCompletion", + action: "RemoveDirectory", + fields: ['crashStage = "reserved"', "progress = 3", 'stage = "done"'], + }, + { + name: "promoted-crash-reachable", + negative: true, + invariant: "NeverPromotedCrashCompletion", + action: "RemoveDirectory", + fields: ['crashStage = "promoted"', "progress = 3", 'stage = "done"'], + }, + { + name: "release-crash-reachable", + negative: true, + invariant: "NeverReleaseCrashCompletion", + action: "RemoveDirectory", + fields: ['crashStage = "empty"', "progress = 3", 'stage = "done"'], + }, + { + name: "unknown-refusal-reachable", + negative: true, + invariant: "NeverUnknownRefusal", + action: "RefuseUnknown", + fields: ['stage = "unknown-retained"', "pending = 2", "savedNext = 0"], + }, + { + name: "fresh-release-retention-reachable", + negative: true, + invariant: "NeverFreshReleaseRetention", + action: "FailFreshRelease", + fields: [ + 'stage = "release-failed"', + "owner = 1", + "directory = TRUE", + "release = FALSE", + ], + }, + ], + }, { name: "native-storage-witness", module: "Witness", diff --git a/tests/models/tla/README.md b/tests/models/tla/README.md index 6fb2fa49e..a24e36b4a 100644 --- a/tests/models/tla/README.md +++ b/tests/models/tla/README.md @@ -19,8 +19,8 @@ No credentials or running VM are needed. ## Native foreground recovery lease takeover `native-frontend-recovery/Recovery.tla` models the explicit saved-run frontend -recovery lease protocol. Its checker and exact exploration-count registration -are pending initial qualification. It bounds one lease to two new candidates, +recovery lease protocol. The positive configuration exhausts **602 distinct +states** (695 generated, maximum depth 22). It bounds one lease to two new candidates, one crash, one immutable resource selection and one possible foreign-binding substitution. Two initial states cover a fresh active recovery at release and a dead selected owner awaiting takeover. The same protocol serves the primary and From 896502a00527da5ee77ef20af865e68dd80d353e Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 17:34:23 -0400 Subject: [PATCH 33/47] test(native): isolate short status deadlines and explicit modes --- tests/native-authored-project-process.test.ts | 82 +++++++++++++++---- tests/native-authored-project-run.test.ts | 5 +- 2 files changed, 72 insertions(+), 15 deletions(-) diff --git a/tests/native-authored-project-process.test.ts b/tests/native-authored-project-process.test.ts index acaf50ada..637833e03 100644 --- a/tests/native-authored-project-process.test.ts +++ b/tests/native-authored-project-process.test.ts @@ -103,6 +103,8 @@ async function fixture( readonly ownerExits?: boolean; readonly splitReady?: boolean; readonly padding?: string; + /** Keep short deadline oracles independent of two interpreter startups. */ + readonly fastControlBoundary?: true; } = {} ) { const root = await mkdtemp(join(tmpdir(), "native-authored-process-")); @@ -159,7 +161,23 @@ async function fixture( ); await writeFile( binary, - `#!/bin/sh\nexec ${quote(process.execPath)} ${quote(script)} "$@"\n` + opts.fastControlBoundary + ? `#!/bin/sh +case " $* " in + *" control "*) + printf status > authenticated-status-started + ${opts.controlKeeper ? `${quote(process.execPath)} -e ${quote('await Bun.sleep(2000); await Bun.write("keeper-complete", "exited");')} &\nprintf '%s' "$!" > keeper-pid\nprintf '%s\\n' synthetic-private-keeper-detail >&2` : ":"} + /bin/sleep ${(opts.controlDelayMs ?? 0) / 1000} + printf '%s\\n' ${quote(JSON.stringify(opts.control ?? status()))} + exit 0 + ;; +esac +trap 'printf cleaned > cleanup-complete; exit 0' TERM +printf '%s' ${quote(first)} +while [ ! -f finish-request ]; do /bin/sleep 0.01; done +printf exited > completion-observed +` + : `#!/bin/sh\nexec ${quote(process.execPath)} ${quote(script)} "$@"\n` ); await chmod(binary, 0o700); return { @@ -410,7 +428,10 @@ test("safe status failure code does not expose private diagnostics or publish", }); test("startup deadline cancels pending authentication before publication and awaits cleanup", async () => { - const opts = await fixture({ controlDelayMs: 3000 }); + const opts = await fixture({ + controlDelayMs: 3000, + fastControlBoundary: true, + }); let published = false; const start = performance.now(); await expect( @@ -435,9 +456,13 @@ test("startup deadline cancels pending authentication before publication and awa }); test("status descendant-held pipes cannot outlive startup admission or owned cleanup", async () => { - const opts = await fixture({ controlKeeper: true }); + const opts = await fixture({ + controlKeeper: true, + fastControlBoundary: true, + }); let published = false; let failure = ""; + let keeper: number | undefined; const start = performance.now(); try { await serveNativeAuthoredProjectGraph({ @@ -458,25 +483,54 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle expect( await Bun.file(join(opts.projectRoot, "cleanup-complete")).text() ).toBe("cleaned"); - const keeper = Number( + keeper = Number( await Bun.file(join(opts.projectRoot, "keeper-pid")).text() ); + expect(Number.isSafeInteger(keeper) && keeper > 1).toBe(true); expect(() => process.kill(keeper, 0)).not.toThrow(); expect( await Bun.file(join(opts.projectRoot, "keeper-complete")).exists() ).toBe(false); } finally { - // The stand-in descendant exits itself; retain its fixture until completion. - const deadline = performance.now() + 4000; - while ( - !(await Bun.file(join(opts.projectRoot, "keeper-complete")).exists()) && - performance.now() < deadline - ) { - await Bun.sleep(25); + if (keeper === undefined) { + // A failed setup never supplies a PID or completion authority. + roots.splice(roots.indexOf(opts.projectRoot), 1); + } else { + const keeperPid = keeper; + const absent = () => { + try { + process.kill(keeperPid, 0); + return false; + } catch (error: unknown) { + return ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ESRCH" + ); + } + }; + // A done file alone does not prove exit. Observe exact PID absence without + // signalling it; any live/reused/unknown PID retains the private fixture. + const deadline = performance.now() + 4000; + while ( + !( + (await Bun.file( + join(opts.projectRoot, "keeper-complete") + ).exists()) && absent() + ) && + performance.now() < deadline + ) { + await Bun.sleep(25); + } + if (!absent()) { + roots.splice(roots.indexOf(opts.projectRoot), 1); + } + expect( + await Bun.file(join(opts.projectRoot, "keeper-complete")).text() + ).toBe("exited"); + expect(absent()).toBe(true); } - expect( - await Bun.file(join(opts.projectRoot, "keeper-complete")).text() - ).toBe("exited"); } }, 6000); diff --git a/tests/native-authored-project-run.test.ts b/tests/native-authored-project-run.test.ts index 0d3d9c38c..d57a5c52b 100644 --- a/tests/native-authored-project-run.test.ts +++ b/tests/native-authored-project-run.test.ts @@ -378,8 +378,11 @@ test("shared internal metadata permissions stay unchanged and branch scopes rema const opts = await fixture(); const internal = join(opts.projectDir, ".internal"); await mkdir(internal, { mode: 0o755 }); + await chmod(internal, 0o755); + const originalMode = (await stat(internal)).mode & 0o777; + expect(originalMode).toBe(0o755); await prepare(opts); - expect((await stat(internal)).mode & 0o777).toBe(0o755); + expect((await stat(internal)).mode & 0o777).toBe(originalMode); const original = await save({ ...opts, record: record() }); const branch = "feature/\u{1F408}"; const other = await save({ ...opts, branch, record: record("9".repeat(32)) }); From 41204e7b549f18b6326b8146e1f7823adf3795bf Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 17:35:28 -0400 Subject: [PATCH 34/47] test(native): retain uncertain keeper completion evidence --- tests/native-authored-project-process.test.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/native-authored-project-process.test.ts b/tests/native-authored-project-process.test.ts index 637833e03..9722d3a4d 100644 --- a/tests/native-authored-project-process.test.ts +++ b/tests/native-authored-project-process.test.ts @@ -523,13 +523,17 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle ) { await Bun.sleep(25); } - if (!absent()) { + const completion = await Bun.file( + join(opts.projectRoot, "keeper-complete") + ) + .text() + .catch(() => undefined); + const keeperAbsent = absent(); + if (completion !== "exited" || !keeperAbsent) { roots.splice(roots.indexOf(opts.projectRoot), 1); } - expect( - await Bun.file(join(opts.projectRoot, "keeper-complete")).text() - ).toBe("exited"); - expect(absent()).toBe(true); + expect(completion).toBe("exited"); + expect(keeperAbsent).toBe(true); } } }, 6000); From cb4f821049399659ef4518e3931acc50b576cc49 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 17:42:12 -0400 Subject: [PATCH 35/47] test(native): record selected status fixture stages --- tests/native-authored-project-process.test.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/native-authored-project-process.test.ts b/tests/native-authored-project-process.test.ts index 9722d3a4d..b71faaff0 100644 --- a/tests/native-authored-project-process.test.ts +++ b/tests/native-authored-project-process.test.ts @@ -163,6 +163,7 @@ async function fixture( binary, opts.fastControlBoundary ? `#!/bin/sh +printf entered > fast-owner-entered case " $* " in *" control "*) printf status > authenticated-status-started @@ -463,11 +464,22 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle let published = false; let failure = ""; let keeper: number | undefined; + let receiptObserved = false; + let ownedExitCode: number | undefined; + let nativeFailureObserved = false; const start = performance.now(); try { await serveNativeAuthoredProjectGraph({ ...opts, startupTimeoutMs: 200, + onReceipt: () => { + receiptObserved = true; + return undefined; + }, + onExitDiagnostic: (diagnostic) => { + ownedExitCode = diagnostic.exitCode; + nativeFailureObserved = diagnostic.nativeCode !== undefined; + }, onReady: async () => { published = true; }, @@ -475,6 +487,28 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle } catch (error) { failure = String(error); } + if (process.env.HACK_NATIVE_STATUS_FIXTURE_DIAGNOSTIC === "1") { + console.error( + JSON.stringify({ + kind: "native-status-fixture-stages", + receiptObserved, + ownedExitCode: ownedExitCode ?? null, + nativeFailureObserved, + ownerEntered: await Bun.file( + join(opts.projectRoot, "fast-owner-entered") + ).exists(), + statusEntered: await Bun.file( + join(opts.projectRoot, "authenticated-status-started") + ).exists(), + keeperSelected: await Bun.file( + join(opts.projectRoot, "keeper-pid") + ).exists(), + cleanupObserved: await Bun.file( + join(opts.projectRoot, "cleanup-complete") + ).exists(), + }) + ); + } try { expect(performance.now() - start).toBeLessThan(1500); expect(failure).toContain("canceled"); From 9d7d662c95dbe84f797e425e658573708d420381 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 17:43:25 -0400 Subject: [PATCH 36/47] test(native): retain fixtures across diagnostic failures --- tests/native-authored-project-process.test.ts | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/tests/native-authored-project-process.test.ts b/tests/native-authored-project-process.test.ts index b71faaff0..34781f1de 100644 --- a/tests/native-authored-project-process.test.ts +++ b/tests/native-authored-project-process.test.ts @@ -487,30 +487,31 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle } catch (error) { failure = String(error); } - if (process.env.HACK_NATIVE_STATUS_FIXTURE_DIAGNOSTIC === "1") { - console.error( - JSON.stringify({ - kind: "native-status-fixture-stages", - receiptObserved, - ownedExitCode: ownedExitCode ?? null, - nativeFailureObserved, - ownerEntered: await Bun.file( - join(opts.projectRoot, "fast-owner-entered") - ).exists(), - statusEntered: await Bun.file( - join(opts.projectRoot, "authenticated-status-started") - ).exists(), - keeperSelected: await Bun.file( - join(opts.projectRoot, "keeper-pid") - ).exists(), - cleanupObserved: await Bun.file( - join(opts.projectRoot, "cleanup-complete") - ).exists(), - }) - ); - } + const operationElapsedMs = performance.now() - start; try { - expect(performance.now() - start).toBeLessThan(1500); + if (process.env.HACK_NATIVE_STATUS_FIXTURE_DIAGNOSTIC === "1") { + console.error( + JSON.stringify({ + kind: "native-status-fixture-stages", + receiptObserved, + ownedExitCode: ownedExitCode ?? null, + nativeFailureObserved, + ownerEntered: await Bun.file( + join(opts.projectRoot, "fast-owner-entered") + ).exists(), + statusEntered: await Bun.file( + join(opts.projectRoot, "authenticated-status-started") + ).exists(), + keeperSelected: await Bun.file( + join(opts.projectRoot, "keeper-pid") + ).exists(), + cleanupObserved: await Bun.file( + join(opts.projectRoot, "cleanup-complete") + ).exists(), + }) + ); + } + expect(operationElapsedMs).toBeLessThan(1500); expect(failure).toContain("canceled"); expect(failure).not.toContain("synthetic-private-keeper-detail"); expect(published).toBe(false); From d1af34d99844cff849a62a82a6d82455238ee847 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 17:53:44 -0400 Subject: [PATCH 37/47] test: admit held status pipes before cancellation --- tests/native-authored-project-process.test.ts | 103 ++++++++++++++---- 1 file changed, 79 insertions(+), 24 deletions(-) diff --git a/tests/native-authored-project-process.test.ts b/tests/native-authored-project-process.test.ts index 34781f1de..516fbc3ec 100644 --- a/tests/native-authored-project-process.test.ts +++ b/tests/native-authored-project-process.test.ts @@ -467,32 +467,80 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle let receiptObserved = false; let ownedExitCode: number | undefined; let nativeFailureObserved = false; - const start = performance.now(); - try { - await serveNativeAuthoredProjectGraph({ - ...opts, - startupTimeoutMs: 200, - onReceipt: () => { - receiptObserved = true; - return undefined; - }, - onExitDiagnostic: (diagnostic) => { - ownedExitCode = diagnostic.exitCode; - nativeFailureObserved = diagnostic.nativeCode !== undefined; - }, - onReady: async () => { - published = true; - }, + let operationSettled = false; + let admitted = false; + let cancellationStarted: number | undefined; + let cancellationTimer: ReturnType | undefined; + const controller = new AbortController(); + const operation = serveNativeAuthoredProjectGraph({ + ...opts, + // Admission is a precondition here; the independent 200ms startup test + // above retains that deadline. Cancel only an observed pending status. + startupTimeoutMs: 2000, + signal: controller.signal, + onReceipt: () => { + receiptObserved = true; + return undefined; + }, + onExitDiagnostic: (diagnostic) => { + ownedExitCode = diagnostic.exitCode; + nativeFailureObserved = diagnostic.nativeCode !== undefined; + }, + onReady: async () => { + published = true; + }, + }) + .catch((error: unknown) => { + failure = String(error); + }) + .finally(() => { + operationSettled = true; }); - } catch (error) { - failure = String(error); - } - const operationElapsedMs = performance.now() - start; try { + const admissionDeadline = performance.now() + 2000; + while (!operationSettled && performance.now() < admissionDeadline) { + const statusEntered = await Bun.file( + join(opts.projectRoot, "authenticated-status-started") + ) + .text() + .catch(() => undefined); + const keeperText = await Bun.file(join(opts.projectRoot, "keeper-pid")) + .text() + .catch(() => undefined); + const selectedKeeper = Number(keeperText); + if ( + receiptObserved && + statusEntered === "status" && + Number.isSafeInteger(selectedKeeper) && + selectedKeeper > 1 + ) { + keeper = selectedKeeper; + try { + process.kill(keeper, 0); + admitted = true; + break; + } catch { + // Unknown or already-dead keeper never supplies admission authority. + } + } + await Bun.sleep(5); + } + if (admitted) { + cancellationStarted = performance.now(); + cancellationTimer = setTimeout(() => controller.abort(), 200); + } else { + controller.abort(); + } + await operation; + const operationElapsedMs = + cancellationStarted === undefined + ? undefined + : performance.now() - cancellationStarted; if (process.env.HACK_NATIVE_STATUS_FIXTURE_DIAGNOSTIC === "1") { console.error( JSON.stringify({ kind: "native-status-fixture-stages", + admitted, receiptObserved, ownedExitCode: ownedExitCode ?? null, nativeFailureObserved, @@ -511,6 +559,11 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle }) ); } + expect(admitted).toBe(true); + if (keeper === undefined) { + throw new Error("Status keeper admission requires an observed live PID."); + } + expect(operationElapsedMs).toBeDefined(); expect(operationElapsedMs).toBeLessThan(1500); expect(failure).toContain("canceled"); expect(failure).not.toContain("synthetic-private-keeper-detail"); @@ -518,16 +571,18 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle expect( await Bun.file(join(opts.projectRoot, "cleanup-complete")).text() ).toBe("cleaned"); - keeper = Number( - await Bun.file(join(opts.projectRoot, "keeper-pid")).text() - ); expect(Number.isSafeInteger(keeper) && keeper > 1).toBe(true); expect(() => process.kill(keeper, 0)).not.toThrow(); expect( await Bun.file(join(opts.projectRoot, "keeper-complete")).exists() ).toBe(false); } finally { - if (keeper === undefined) { + clearTimeout(cancellationTimer); + if (!operationSettled) { + controller.abort(); + } + await operation; + if (!admitted || keeper === undefined) { // A failed setup never supplies a PID or completion authority. roots.splice(roots.indexOf(opts.projectRoot), 1); } else { From 56c3b18d527ffc4aa96fa68cf6feb8415d72a6cd Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 17:55:05 -0400 Subject: [PATCH 38/47] test: bind pending status cancellation witness --- tests/native-authored-project-process.test.ts | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/native-authored-project-process.test.ts b/tests/native-authored-project-process.test.ts index 516fbc3ec..9e98bc95f 100644 --- a/tests/native-authored-project-process.test.ts +++ b/tests/native-authored-project-process.test.ts @@ -461,6 +461,9 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle controlKeeper: true, fastControlBoundary: true, }); + // Keep the fixture even if admission/settlement outlives the outer test. + // Cleanup becomes eligible only after its exact keeper completion+absence. + roots.splice(roots.indexOf(opts.projectRoot), 1); let published = false; let failure = ""; let keeper: number | undefined; @@ -469,6 +472,7 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle let nativeFailureObserved = false; let operationSettled = false; let admitted = false; + let cancellationRequested = false; let cancellationStarted: number | undefined; let cancellationTimer: ReturnType | undefined; const controller = new AbortController(); @@ -527,7 +531,12 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle } if (admitted) { cancellationStarted = performance.now(); - cancellationTimer = setTimeout(() => controller.abort(), 200); + cancellationTimer = setTimeout(() => { + if (!operationSettled) { + cancellationRequested = true; + controller.abort(); + } + }, 200); } else { controller.abort(); } @@ -541,6 +550,7 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle JSON.stringify({ kind: "native-status-fixture-stages", admitted, + cancellationRequested, receiptObserved, ownedExitCode: ownedExitCode ?? null, nativeFailureObserved, @@ -560,6 +570,7 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle ); } expect(admitted).toBe(true); + expect(cancellationRequested).toBe(true); if (keeper === undefined) { throw new Error("Status keeper admission requires an observed live PID."); } @@ -582,10 +593,7 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle controller.abort(); } await operation; - if (!admitted || keeper === undefined) { - // A failed setup never supplies a PID or completion authority. - roots.splice(roots.indexOf(opts.projectRoot), 1); - } else { + if (admitted && keeper !== undefined) { const keeperPid = keeper; const absent = () => { try { @@ -619,8 +627,8 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle .text() .catch(() => undefined); const keeperAbsent = absent(); - if (completion !== "exited" || !keeperAbsent) { - roots.splice(roots.indexOf(opts.projectRoot), 1); + if (completion === "exited" && keeperAbsent) { + roots.push(opts.projectRoot); } expect(completion).toBe("exited"); expect(keeperAbsent).toBe(true); From 936ecce8a279df142334772478ab5fb1fdf5524d Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 18:33:15 -0400 Subject: [PATCH 39/47] test: observe native startup deadline phases --- tests/native-authored-project-process.test.ts | 91 +++++++++++++++---- 1 file changed, 72 insertions(+), 19 deletions(-) diff --git a/tests/native-authored-project-process.test.ts b/tests/native-authored-project-process.test.ts index 9e98bc95f..5b34db3b1 100644 --- a/tests/native-authored-project-process.test.ts +++ b/tests/native-authored-project-process.test.ts @@ -433,27 +433,80 @@ test("startup deadline cancels pending authentication before publication and awa controlDelayMs: 3000, fastControlBoundary: true, }); + // Preserve uncertain phase evidence even if the operation or test never settles. + roots.splice(roots.indexOf(opts.projectRoot), 1); let published = false; + let receiptObserved = false; + let ownedExitCode: number | undefined; + let nativeFailureObserved = false; + let failure: unknown; const start = performance.now(); - await expect( - serveNativeAuthoredProjectGraph({ - ...opts, - startupTimeoutMs: 200, - onReady: async () => { - published = true; - }, - }) - ).rejects.toThrow("canceled"); - expect(performance.now() - start).toBeLessThan(1500); - expect(published).toBe(false); - expect( - await Bun.file( - join(opts.projectRoot, "authenticated-status-started") - ).text() - ).toBe("status"); - expect( - await Bun.file(join(opts.projectRoot, "cleanup-complete")).text() - ).toBe("cleaned"); + await serveNativeAuthoredProjectGraph({ + ...opts, + startupTimeoutMs: 200, + onReceipt: () => { + receiptObserved = true; + return undefined; + }, + onExitDiagnostic: (diagnostic) => { + ownedExitCode = diagnostic.exitCode; + nativeFailureObserved = diagnostic.nativeCode !== undefined; + }, + onReady: async () => { + published = true; + }, + }).catch((error: unknown) => { + failure = error; + }); + const elapsedMs = performance.now() - start; + let ownerEntered = false; + let statusEntered = false; + let cleanupObserved = false; + try { + ownerEntered = + (await Bun.file(join(opts.projectRoot, "fast-owner-entered")) + .text() + .catch(() => undefined)) === "entered"; + statusEntered = + (await Bun.file(join(opts.projectRoot, "authenticated-status-started")) + .text() + .catch(() => undefined)) === "status"; + cleanupObserved = + (await Bun.file(join(opts.projectRoot, "cleanup-complete")) + .text() + .catch(() => undefined)) === "cleaned"; + if (process.env.HACK_NATIVE_STARTUP_FIXTURE_DIAGNOSTIC === "1") { + console.error( + JSON.stringify({ + kind: "native-startup-fixture-observation", + elapsedMs, + receiptObserved, + ownedExitCode: ownedExitCode ?? null, + nativeFailureObserved, + ownerEntered, + statusEntered, + cleanupObserved, + }) + ); + } + expect(failure).toBeInstanceOf(Error); + expect(String(failure)).toContain("canceled"); + expect(elapsedMs).toBeLessThan(1500); + expect(published).toBe(false); + expect(statusEntered).toBe(true); + expect(cleanupObserved).toBe(true); + } finally { + if ( + ownerEntered && + statusEntered && + cleanupObserved && + receiptObserved && + ownedExitCode === 0 && + !nativeFailureObserved + ) { + roots.push(opts.projectRoot); + } + } }); test("status descendant-held pipes cannot outlive startup admission or owned cleanup", async () => { From 38eb404f8d6091edb4aeb1004ea698db6439b659 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 18:45:12 -0400 Subject: [PATCH 40/47] test: bind startup deadline checks to observed phases --- tests/native-authored-project-process.test.ts | 70 ++++++++++++++----- 1 file changed, 54 insertions(+), 16 deletions(-) diff --git a/tests/native-authored-project-process.test.ts b/tests/native-authored-project-process.test.ts index 5b34db3b1..9d4164d15 100644 --- a/tests/native-authored-project-process.test.ts +++ b/tests/native-authored-project-process.test.ts @@ -8,6 +8,7 @@ import { parseNativeAuthoredReview, } from "../src/backends/native-authored-graph-protocol.ts"; import { serveNativeAuthoredProjectGraph } from "../src/backends/native-project-process.ts"; +import { NativeRuntimeRequestError } from "../src/backends/native-runtime-client.ts"; const roots: string[] = []; const run = "a".repeat(32); @@ -428,7 +429,7 @@ test("safe status failure code does not expose private diagnostics or publish", ).toBe("cleaned"); }); -test("startup deadline cancels pending authentication before publication and awaits cleanup", async () => { +test("startup deadline refuses publication and settles its observed startup prefix", async () => { const opts = await fixture({ controlDelayMs: 3000, fastControlBoundary: true, @@ -462,19 +463,32 @@ test("startup deadline cancels pending authentication before publication and awa let ownerEntered = false; let statusEntered = false; let cleanupObserved = false; + const marker = async (name: string): Promise => + await Bun.file(join(opts.projectRoot, name)) + .text() + .catch((error: unknown) => { + if ( + error !== null && + typeof error === "object" && + "code" in error && + error.code === "ENOENT" + ) { + return undefined; + } + throw error; + }); try { - ownerEntered = - (await Bun.file(join(opts.projectRoot, "fast-owner-entered")) - .text() - .catch(() => undefined)) === "entered"; - statusEntered = - (await Bun.file(join(opts.projectRoot, "authenticated-status-started")) - .text() - .catch(() => undefined)) === "status"; - cleanupObserved = - (await Bun.file(join(opts.projectRoot, "cleanup-complete")) - .text() - .catch(() => undefined)) === "cleaned"; + const ownerMarker = await marker("fast-owner-entered"); + const statusMarker = await marker("authenticated-status-started"); + const cleanupMarker = await marker("cleanup-complete"); + expect(ownerMarker === undefined || ownerMarker === "entered").toBe(true); + expect(statusMarker === undefined || statusMarker === "status").toBe(true); + expect(cleanupMarker === undefined || cleanupMarker === "cleaned").toBe( + true + ); + ownerEntered = ownerMarker === "entered"; + statusEntered = statusMarker === "status"; + cleanupObserved = cleanupMarker === "cleaned"; if (process.env.HACK_NATIVE_STARTUP_FIXTURE_DIAGNOSTIC === "1") { console.error( JSON.stringify({ @@ -490,11 +504,35 @@ test("startup deadline cancels pending authentication before publication and awa ); } expect(failure).toBeInstanceOf(Error); - expect(String(failure)).toContain("canceled"); + const failureMessage = + failure instanceof Error ? failure.message : undefined; expect(elapsedMs).toBeLessThan(1500); expect(published).toBe(false); - expect(statusEntered).toBe(true); - expect(cleanupObserved).toBe(true); + expect(nativeFailureObserved).toBe(false); + if (receiptObserved) { + // Ready is observed synchronously before status admission. The deadline + // can interrupt before or after the status child enters its first command. + expect(ownerEntered).toBe(true); + expect(cleanupObserved).toBe(true); + expect(ownedExitCode).toBe(0); + expect(failure).toBeInstanceOf(NativeRuntimeRequestError); + expect([ + "Native runtime request was canceled before admission; no request was started.", + "Native runtime request was canceled; its outcome may be uncertain. No request was replayed.", + ]).toContain(failureMessage); + } else { + // The deadline also owns prefixes before Ready is consumed: no first + // command, owner-only, or a trapped owner whose Ready frame is unread. + // These cannot claim pending authentication or publication authority. + expect(statusEntered).toBe(false); + expect(failureMessage).toBe( + "Native graph startup was interrupted or failed; inspect owned state before retrying." + ); + expect(ownedExitCode).toBe(cleanupObserved ? 0 : 143); + if (cleanupObserved) { + expect(ownerEntered).toBe(true); + } + } } finally { if ( ownerEntered && From a02622a5be39bc8a7b277ed28db81d773c24a86e Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 18:46:43 -0400 Subject: [PATCH 41/47] test: preserve buffered ready deadline refusal --- tests/native-authored-project-process.test.ts | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/native-authored-project-process.test.ts b/tests/native-authored-project-process.test.ts index 9d4164d15..a6111ed65 100644 --- a/tests/native-authored-project-process.test.ts +++ b/tests/native-authored-project-process.test.ts @@ -525,12 +525,24 @@ test("startup deadline refuses publication and settles its observed startup pref // command, owner-only, or a trapped owner whose Ready frame is unread. // These cannot claim pending authentication or publication authority. expect(statusEntered).toBe(false); - expect(failureMessage).toBe( - "Native graph startup was interrupted or failed; inspect owned state before retrying." - ); - expect(ownedExitCode).toBe(cleanupObserved ? 0 : 143); - if (cleanupObserved) { + if ( + failureMessage === + "Native graph readiness identity is invalid or canceled." + ) { + // A complete buffered frame can cross the deadline before the + // pre-callback fence admits it to onReceipt. The TERM trap was installed + // before that frame, but no status or ready publication was admitted. expect(ownerEntered).toBe(true); + expect(cleanupObserved).toBe(true); + expect(ownedExitCode).toBe(0); + } else { + expect(failureMessage).toBe( + "Native graph startup was interrupted or failed; inspect owned state before retrying." + ); + expect(ownedExitCode).toBe(cleanupObserved ? 0 : 143); + if (cleanupObserved) { + expect(ownerEntered).toBe(true); + } } } } finally { From 329828e63f75380bfa3e042e93e182f806a323bc Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 18:55:03 -0400 Subject: [PATCH 42/47] test: narrow observed startup error and keeper identity --- tests/native-authored-project-process.test.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/native-authored-project-process.test.ts b/tests/native-authored-project-process.test.ts index a6111ed65..65f983612 100644 --- a/tests/native-authored-project-process.test.ts +++ b/tests/native-authored-project-process.test.ts @@ -504,8 +504,10 @@ test("startup deadline refuses publication and settles its observed startup pref ); } expect(failure).toBeInstanceOf(Error); - const failureMessage = - failure instanceof Error ? failure.message : undefined; + if (!(failure instanceof Error)) { + throw new Error("Startup fixture requires its observed Error."); + } + const failureMessage = failure.message; expect(elapsedMs).toBeLessThan(1500); expect(published).toBe(false); expect(nativeFailureObserved).toBe(false); @@ -677,6 +679,7 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle if (keeper === undefined) { throw new Error("Status keeper admission requires an observed live PID."); } + const admittedKeeper = keeper; expect(operationElapsedMs).toBeDefined(); expect(operationElapsedMs).toBeLessThan(1500); expect(failure).toContain("canceled"); @@ -685,8 +688,10 @@ test("status descendant-held pipes cannot outlive startup admission or owned cle expect( await Bun.file(join(opts.projectRoot, "cleanup-complete")).text() ).toBe("cleaned"); - expect(Number.isSafeInteger(keeper) && keeper > 1).toBe(true); - expect(() => process.kill(keeper, 0)).not.toThrow(); + expect(Number.isSafeInteger(admittedKeeper) && admittedKeeper > 1).toBe( + true + ); + expect(() => process.kill(admittedKeeper, 0)).not.toThrow(); expect( await Bun.file(join(opts.projectRoot, "keeper-complete")).exists() ).toBe(false); From e6ad592377f56b6147dc74ba4a3fbb10d9edbde9 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 19:27:32 -0400 Subject: [PATCH 43/47] test: expose dead relay publication refusal predicates --- .../provider/relay_owner/publication/tests.rs | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/packages/runtime-core/src/provider/relay_owner/publication/tests.rs b/packages/runtime-core/src/provider/relay_owner/publication/tests.rs index ebbfc4a9b..bd241da0d 100644 --- a/packages/runtime-core/src/provider/relay_owner/publication/tests.rs +++ b/packages/runtime-core/src/provider/relay_owner/publication/tests.rs @@ -2,7 +2,7 @@ use super::*; use crate::provider::{host_endpoint::HostEndpoint, relay_loop::Limits, relay_owner::OwnerLimits}; use std::{ net::TcpListener, - os::unix::net::UnixStream, + os::unix::{net::UnixStream, process::ExitStatusExt}, process::{Child, Command, Stdio}, sync::atomic::{AtomicU64, Ordering}, }; @@ -292,7 +292,8 @@ fn dead_publication_recovery_requires_unchanged_receipt_and_socket() { #[cfg(target_os = "macos")] assert!(pin.verify_dead().is_err()); child.0.kill().unwrap(); - child.0.wait().unwrap(); + let exit = child.0.wait().unwrap(); + assert_eq!(exit.signal(), Some(libc::SIGKILL)); let mut replacement = None; if replace_socket { fs::remove_file(&pin.paths.socket).unwrap(); @@ -307,7 +308,22 @@ fn dead_publication_recovery_requires_unchanged_receipt_and_socket() { assert!(pin.paths.receipt.exists()); } else { #[cfg(target_os = "macos")] - pin.verify_dead().unwrap(); + { + let dead = pin.verify_dead(); + if dead.is_err() { + let receipt_unchanged = pin.verify_receipt().is_ok(); + let socket_unchanged = pin.verify_socket().is_ok(); + let process_absent = match identity::alive(pin.receipt.process.pid) { + Ok(false) => "true", + Ok(true) => "false", + Err(_) => "unknown", + }; + panic!( + "Dead relay publication refused: receipt_unchanged={receipt_unchanged} socket_unchanged={socket_unchanged} process_absent={process_absent} child_signal={:?}", + exit.signal() + ); + } + } pin.recover().unwrap(); pin.recover().unwrap(); assert!(absent(&pin.paths.socket).unwrap()); From cd2df7fdd4aab3f990ab3d783040adfbee74345c Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 20:13:44 -0400 Subject: [PATCH 44/47] fix(native): bind foreground owners to stable boot sessions --- docs/reference/native-authored-commands.md | 10 +- packages/runtime-core/README.md | 33 +++- .../src/provider/graph/native/foreground.rs | 1 + .../graph/native/foreground/host_boot.rs | 148 ++++++++++++++++++ .../provider/graph/native/foreground/owner.rs | 137 +++++++++++++--- .../graph/native/foreground/recovery.rs | 24 ++- .../graph/native/foreground/recovery/tests.rs | 63 +++++++- .../provider/graph/native/foreground/tests.rs | 122 ++++++++++++++- .../native-authored-recovery-protocol.ts | 45 ++++-- .../native-authored-project-recovery.test.ts | 23 ++- .../native-authored-recovery-protocol.test.ts | 49 ++++++ 11 files changed, 588 insertions(+), 67 deletions(-) create mode 100644 packages/runtime-core/src/provider/graph/native/foreground/host_boot.rs diff --git a/docs/reference/native-authored-commands.md b/docs/reference/native-authored-commands.md index 71b378eed..9723f2ff5 100644 --- a/docs/reference/native-authored-commands.md +++ b/docs/reference/native-authored-commands.md @@ -38,12 +38,18 @@ and refuses replay; a missing ready mapping does not authorize a fresh start. `down --recover` explicitly retires a complete stored native generation whose foreground publisher and frontend admission owner are both dead on the same host -boot. It requires the current version 3 native publication, the exact stored Ready, +boot. New version 4 native publications use the kernel boot-session UUID, which +remains independent of calendar clock correction. Recovery requires the exact stored Ready, startup intent and private source envelope, and the original guest incarnation. Live owners, older version 2 dead publications, partial startup, pending writes, changed files and rebooted guests refuse. Env/profile changes, service subsets and `--json` are unsupported for this operation. It neither recompiles authored input nor acquires managed values or starts a provider. +The closed version 3 publication and version 1 recovery selector remain supported +with their original calendar boot qualifier; clock drift conservatively refuses +that legacy recovery. New version 4 owners use version 2 UUID selectors. Missing, +malformed or unavailable UUIDs refuse; no old record is migrated or given inferred +session authority. PID birth/executable, peer, inode, gate and run-lock checks remain. Recovery commits a distinct private intent before calling the Rust cleanup owner. The original raw selectors and resource inventory remain fixed through retries; @@ -67,4 +73,4 @@ this operation does not repair arbitrary partial lock or file publications. The native receipt and source paths are distinct from strict Compose v1 artifacts. No native hash substitutes for a normalized Compose hash. Source and fake-driver checks do not qualify an installed frontend, a live provider, the full authored -corpus, actual version 3 dead-owner recovery or resource overhead; those remain separate gates. +corpus, actual dead-owner recovery or resource overhead; those remain separate gates. diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 3570fd6cf..2ed03adad 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -114,16 +114,21 @@ this same subset. Its private `native-graph-owner` record and authenticated v2 `native-graph-control` status/cleanup protocol bind the native review, exact process incarnation, directory, socket, file and retained per-run lock. Direct run/cleanup and foreground publication exclude each other under that lock. -New publications use a closed version3 owner record with independently captured -native host boot time. Live authentication rechecks that boot together with the -existing exact process and filesystem identities. The closed version2 live-owner -decoder remains available without inferring host boot from PID birth or guest -receipt boot. Version2 records reject the new field, and version3 records require -it. Native runtime receipts and the authenticated control/ready wire remain v2; +New publications use a closed version4 owner with `host_boot_uuid`, captured by +the native read-only `kern.bootsessionuuid` sysctl. Its fixed 37-byte response must +contain a nonzero ASCII UUID and terminal NUL; the reader canonicalizes lowercase. +Persisted UUIDs must already be lowercase. Unavailable or malformed readings refuse, +without a timestamp or shell fallback. Live authentication rechecks the session +together with the exact nonzero process incarnation and filesystem identities; +it does not compare process birth against mutable calendar boot time. +The closed version2 live-owner decoder remains available without a boot qualifier. +Version3 remains strictly qualified by its original `host_boot_micros`; it receives +no inferred UUID or migration. Each version rejects the other versions' qualifiers. +Native runtime receipts and the authenticated control/ready wire remain v2; Compose receipt and owner formats remain unchanged. This provenance alone grants no dead-owner recovery authority. The inactive read-only recovery selector admits only a complete Ready journal and -a dead version3 publication on the same native host boot. It captures the current +a dead version3 or version4 publication on the same qualified native host boot. It captures the current private owner bytes and inode; its raw SHA is a selector for later independent admission, not an external identity anchor from before that capture. After a durable intent, the saved inode and digest refuse replacement. Cleanup progress @@ -131,12 +136,24 @@ retains the original Ready selectors and exact inventory. Pending writes refuse; retired paths require the intent's preceding phase and unchanged archived inode. `graph native recovery-selection --run-id ID --json` exposes only the closed selection: run, original value-free Ready receipt, raw receipt/owner hashes and -native host boot time. The receipt contains review hashes, workload readiness, +the original boot qualifier. Version1 selections retain version3 calendar micros; +version2 selections carry version4 `host_boot_uuid`. Neither rewrites original +owner bytes, inode or digest. Version2 dead owners remain ineligible. Version3 +recovery conservatively refuses if its calendar qualifier has drifted. +The receipt contains review hashes, workload readiness, resource IDs/images/networks and terminal observations; it contains no environment keys/values, command argv, source bytes or publication process/path metadata. The selector creates no intent and connects no provider. It grants no cleanup or frontend recovery authority. +The UUID boundary addresses calendar correction without weakening reboot refusal. +[XNU calendar updates adjust boot time](https://github.com/apple-oss-distributions/xnu/blob/main/osfmk/kern/clock.c#L674-L752), +while the [read-only boot-session sysctl](https://github.com/apple-oss-distributions/xnu/blob/main/bsd/kern/kern_sysctl.c#L2735-L2762) +is [generated at boot](https://github.com/apple-oss-distributions/xnu/blob/main/iokit/Kernel/IOPMrootDomain.cpp#L3846-L3861). +Unrelated legacy filesystem chronology still uses its existing calendar helper. +Injected-reader and codec regressions qualify the ownership fences without changing +the host clock or rebooting. They do not establish actual provider recovery. + The separate explicit library recovery entrypoint re-admits both raw selectors and commits a bounded 64 KiB recovery intent before cleanup. It retains the publication gate, original operation lock and cleanup provider lease through diff --git a/packages/runtime-core/src/provider/graph/native/foreground.rs b/packages/runtime-core/src/provider/graph/native/foreground.rs index 55161dd75..9d51f4fd0 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground.rs @@ -3,6 +3,7 @@ use super::*; use crate::provider::graph::foreground::{signals, transport}; use serde::{Deserialize, Serialize}; use std::{cell::Cell, io::Write}; +mod host_boot; mod owner; pub mod recovery; pub(super) use owner::DirectGuard; diff --git a/packages/runtime-core/src/provider/graph/native/foreground/host_boot.rs b/packages/runtime-core/src/provider/graph/native/foreground/host_boot.rs new file mode 100644 index 000000000..83457d9c6 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/native/foreground/host_boot.rs @@ -0,0 +1,148 @@ +//! Stable native host boot authority. Calendar boot time is not an incarnation ID. +use super::{CandidateError, refused}; +use serde::{Deserialize, Deserializer, Serialize}; + +/// Persisted boot qualifiers are canonical, nonzero UUIDs. No timestamp fallback. +#[derive(Clone, PartialEq, Eq, Serialize)] +#[serde(transparent)] +pub(super) struct Session(String); +impl Session { + fn parse(value: &str) -> Option { + let bytes = value.as_bytes(); + if bytes.len() != 36 + || bytes.iter().enumerate().any(|(i, byte)| { + if matches!(i, 8 | 13 | 18 | 23) { + *byte != b'-' + } else { + !byte.is_ascii_digit() && !(b'a'..=b'f').contains(byte) + } + }) + || !bytes + .iter() + .any(|byte| matches!(byte, b'1'..=b'9' | b'a'..=b'f')) + { + return None; + } + Some(Self(value.into())) + } +} +impl<'de> Deserialize<'de> for Session { + fn deserialize>(reader: D) -> Result { + let value = String::deserialize(reader)?; + Self::parse(&value).ok_or_else(|| serde::de::Error::custom("Invalid host boot qualifier")) + } +} + +fn decode(buffer: &[u8; 37], length: usize) -> Result { + if length != buffer.len() || buffer[36] != 0 { + return Err(refused()); + } + let text = std::str::from_utf8(&buffer[..36]).map_err(|_| refused())?; + // XNU publishes uppercase UUID text; only the native reader normalizes it. + Session::parse(&text.to_ascii_lowercase()).ok_or_else(refused) +} + +pub(super) fn read() -> Result { + #[cfg(test)] + if let Some(value) = test::next() { + return value.ok_or_else(refused); + } + let mut buffer = [0u8; 37]; + let mut length = buffer.len(); + // SAFETY: the read-only sysctl writes at most this fixed initialized buffer; + // its size pointer is live, newp is null, and no pointer is retained. + if unsafe { + libc::sysctlbyname( + c"kern.bootsessionuuid".as_ptr(), + buffer.as_mut_ptr().cast(), + &mut length, + std::ptr::null_mut(), + 0, + ) + } != 0 + { + return Err(refused()); + } + decode(&buffer, length) +} + +#[cfg(test)] +pub(super) mod test { + use super::*; + use std::{cell::RefCell, collections::VecDeque}; + thread_local! { + static OVERRIDE: RefCell>>> = const { RefCell::new(None) }; + } + pub(super) fn next() -> Option> { + OVERRIDE.with(|cell| { + let mut state = cell.borrow_mut(); + let values = state.as_mut()?; + if values.len() > 1 { + values.pop_front() + } else { + values.front().cloned() + } + }) + } + pub(in super::super) struct Guard(Option>>); + impl Guard { + pub(in super::super) fn set(value: Option<&str>) -> Self { + Self::sequence(&[value]) + } + pub(in super::super) fn sequence(values: &[Option<&str>]) -> Self { + assert!(!values.is_empty()); + Self(OVERRIDE.with(|cell| { + cell.replace(Some( + values + .iter() + .map(|value| value.map(|v| Session::parse(v).unwrap())) + .collect(), + )) + })) + } + } + impl Drop for Guard { + fn drop(&mut self) { + OVERRIDE.with(|cell| cell.replace(self.0.take())); + } + } + #[test] + fn fixed_native_buffer_normalizes_only_valid_exact_uuid() { + let mut buffer = *b"12345678-ABCD-ABCD-ABCD-123456789ABC\0"; + assert!( + decode(&buffer, 37).unwrap() + == Session::parse("12345678-abcd-abcd-abcd-123456789abc").unwrap() + ); + for length in [0, 36, 38] { + assert!(decode(&buffer, length).is_err()); + } + buffer[36] = b'x'; + assert!(decode(&buffer, 37).is_err()); + for value in [ + "00000000-0000-0000-0000-000000000000", + "12345678-abcd-abcd-abcd-123456789abg", + "12345678_abcd-abcd-abcd-123456789abc", + ] { + let bytes: [u8; 37] = format!("{value}\0").into_bytes().try_into().unwrap(); + assert!(decode(&bytes, 37).is_err()); + } + buffer = *b"12345678-ABCD-ABCD-ABCD-123456789ABC\0"; + buffer[0] = 0xff; + assert!(decode(&buffer, 37).is_err()); + buffer[0] = 0; + assert!(decode(&buffer, 37).is_err()); + } + #[test] + fn persisted_uuid_is_closed_and_reader_unavailability_never_falls_back() { + for value in [ + serde_json::Value::Null, + serde_json::json!(1), + serde_json::json!("12345678-ABCD-ABCD-ABCD-123456789ABC"), + serde_json::json!("00000000-0000-0000-0000-000000000000"), + ] { + assert!(serde_json::from_value::(value).is_err()); + } + let _guard = Guard::set(None); + assert!(read().is_err()); + } +} diff --git a/packages/runtime-core/src/provider/graph/native/foreground/owner.rs b/packages/runtime-core/src/provider/graph/native/foreground/owner.rs index 0d1431a44..1f495fcd9 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/owner.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/owner.rs @@ -48,7 +48,7 @@ struct BootQualifiedRecord { host_boot_micros: u64, } impl BootQualifiedRecord { - fn into_record(self) -> (Record, Option) { + fn into_record(self) -> (Record, Option) { ( Record { version: self.version, @@ -60,11 +60,52 @@ impl BootQualifiedRecord { socket: self.socket, lock: self.lock, }, - Some(self.host_boot_micros), + Some(HostBoot::LegacyMicros(self.host_boot_micros)), ) } } -fn decode(bytes: &[u8]) -> Result<(Record, Option), CandidateError> { +/// Version4 has no calendar qualifier: exact process incarnation remains a separate fence. +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct SessionQualifiedRecord { + version: u32, + kind: Kind, + candidate: PathBuf, + review: native_input::Review, + process: ProcessIdentity, + parent: (u64, u64), + socket: (u64, u64), + lock: (u64, u64), + host_boot_uuid: host_boot::Session, +} +#[derive(Clone, PartialEq, Eq)] +pub(super) enum HostBoot { + LegacyMicros(u64), + Session(host_boot::Session), +} +impl HostBoot { + fn verify_with( + &self, + micros: impl FnOnce() -> Result, + session: impl FnOnce() -> Result, + ) -> Result<(), CandidateError> { + let equal = match self { + Self::LegacyMicros(expected) => micros().ok() == Some(*expected), + Self::Session(expected) => session().ok().as_ref() == Some(expected), + }; + if !equal { + return Err(refused()); + } + Ok(()) + } + fn verify(&self) -> Result<(), CandidateError> { + self.verify_with( + crate::provider::host_filesystem::host_boot_micros, + host_boot::read, + ) + } +} +fn decode(bytes: &[u8]) -> Result<(Record, Option), CandidateError> { let value: Value = serde_json::from_slice(bytes).map_err(|_| refused())?; match value.get("version").and_then(Value::as_u64) { Some(2) => { @@ -80,6 +121,26 @@ fn decode(bytes: &[u8]) -> Result<(Record, Option), CandidateError> { } Ok(record.into_record()) } + Some(4) => { + let record: SessionQualifiedRecord = + serde_json::from_slice(bytes).map_err(|_| refused())?; + if record.process.start_micros == 0 { + return Err(refused()); + } + Ok(( + Record { + version: record.version, + kind: record.kind, + candidate: record.candidate, + review: record.review, + process: record.process, + parent: record.parent, + socket: record.socket, + lock: record.lock, + }, + Some(HostBoot::Session(record.host_boot_uuid)), + )) + } _ => Err(refused()), } } @@ -160,16 +221,14 @@ impl<'a> DirectGuard<'a> { pub(super) struct Pin { root: PathBuf, record: Record, - host_boot_micros: Option, + host_boot: Option, bytes: Vec, file: (u64, u64), } impl Pin { fn verify_host_boot(&self) -> Result<(), CandidateError> { - if self.host_boot_micros.is_some_and(|boot| { - crate::provider::host_filesystem::host_boot_micros().ok() != Some(boot) - }) { - return Err(refused()); + if let Some(boot) = &self.host_boot { + boot.verify()?; } Ok(()) } @@ -183,7 +242,7 @@ impl Pin { state::check_private_directory(&root).map_err(|_| refused())?; let file = fs::symlink_metadata(root.join("owner.json")).map_err(|_| refused())?; let bytes = native_input::read_file(&root.join("owner.json"), 8192)?; - let (record, host_boot_micros) = decode(&bytes)?; + let (record, host_boot) = decode(&bytes)?; record .review .validate(record.review.scope()) @@ -194,7 +253,7 @@ impl Pin { let pin = Self { root, record, - host_boot_micros, + host_boot, bytes, file: id(&file), }; @@ -284,7 +343,11 @@ impl RecoverySelection { pub(super) fn fingerprint(&self) -> String { format!("{:x}", Sha256::digest(self.bytes.as_bytes())) } - fn record(&self, candidate: &Candidate, run: &str) -> Result<(Record, u64), CandidateError> { + fn record( + &self, + candidate: &Candidate, + run: &str, + ) -> Result<(Record, HostBoot), CandidateError> { if self.bytes.is_empty() || self.bytes.len() > 8192 { return Err(refused()); } @@ -295,14 +358,14 @@ impl RecoverySelection { .validate(record.review.scope()) .map_err(|_| refused())?; // SAFETY: geteuid has no preconditions; private state remains same-user. - if record.version != 3 + if !matches!(record.version, 3 | 4) || record.candidate != candidate.checkout || record.review.scope().run != run || record.process.uid != unsafe { libc::geteuid() } - || crate::provider::host_filesystem::host_boot_micros().ok() != Some(boot) { return Err(refused()); } + boot.verify()?; Ok((record, boot)) } } @@ -331,7 +394,7 @@ impl<'a> RecoveryLease<'a> { Some(value) => value, None => { let pin = Pin::read(candidate, run)?; - if pin.host_boot_micros.is_none() { + if pin.host_boot.is_none() { return Err(refused()); } RecoverySelection { @@ -359,7 +422,7 @@ impl<'a> RecoveryLease<'a> { .record(self.candidate, self.run) .map(|(record, _)| record.review) } - pub(super) fn host_boot_micros(&self) -> Result { + pub(super) fn host_boot(&self) -> Result { self.selection .record(self.candidate, self.run) .map(|(_, boot)| boot) @@ -484,8 +547,7 @@ impl Publication { review: &native_input::Review, ) -> Result { review.validate(review.scope()).map_err(|_| refused())?; - let host_boot_micros = - crate::provider::host_filesystem::host_boot_micros().map_err(|_| refused())?; + let host_boot_uuid = host_boot::read()?; let gate = super::super::super::publication_gate::Guard::acquire(candidate)?; let run = review.scope().run; if exists(&super::super::journal::directory(candidate, run)?)? { @@ -501,8 +563,8 @@ impl Publication { fs::set_permissions(root.join("control.sock"), fs::Permissions::from_mode(0o600)) .map_err(|_| refused())?; listener.set_nonblocking(true).map_err(|_| refused())?; - let record = BootQualifiedRecord { - version: 3, + let record = SessionQualifiedRecord { + version: 4, kind: Kind::NativeGraphOwner, candidate: candidate.checkout.clone(), review: review.clone(), @@ -510,13 +572,13 @@ impl Publication { parent: id(&fs::symlink_metadata(&root).map_err(|_| refused())?), socket: id(&fs::symlink_metadata(root.join("control.sock")).map_err(|_| refused())?), lock: lock.identity().map_err(|_| refused())?, - host_boot_micros, + host_boot_uuid, }; let bytes = serde_json::to_vec(&record).map_err(|_| refused())?; if bytes.len() > 8192 { return Err(refused()); } - let (record, host_boot_micros) = decode(&bytes)?; + let (record, host_boot) = decode(&bytes)?; let mut file = OpenOptions::new() .write(true) .create_new(true) @@ -530,7 +592,7 @@ impl Publication { let pin = Pin { root, record, - host_boot_micros, + host_boot, bytes, file: id(&file.metadata().map_err(|_| refused())?), }; @@ -583,3 +645,34 @@ impl Publication { .map_err(|_| refused()) } } + +#[cfg(test)] +mod boot_tests { + use super::*; + const UUID: &str = "12345678-abcd-abcd-abcd-123456789abc"; + fn session(value: &str) -> host_boot::Session { + serde_json::from_value(json!(value)).unwrap() + } + #[test] + fn session_authority_ignores_calendar_adjustment_but_not_reboot_or_unavailability() { + let boot = HostBoot::Session(session(UUID)); + for changed_wall in [1, u64::MAX] { + // Even an unavailable calendar reader cannot become V4 authority. + boot.verify_with(|| Ok(changed_wall), || Ok(session(UUID))) + .unwrap(); + boot.verify_with(|| Err(refused()), || Ok(session(UUID))) + .unwrap(); + } + assert!( + boot.verify_with( + || Ok(1), + || Ok(session("12345678-abcd-abcd-abcd-123456789abd")) + ) + .is_err() + ); + assert!(boot.verify_with(|| Ok(1), || Err(refused())).is_err()); + let legacy = HostBoot::LegacyMicros(1); + legacy.verify_with(|| Ok(1), || Err(refused())).unwrap(); + assert!(legacy.verify_with(|| Ok(2), || Ok(session(UUID))).is_err()); + } +} diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs index 3bf34dbf1..f5a691d27 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery.rs @@ -182,7 +182,15 @@ pub struct Selection { receipt: Receipt, receipt_sha256: String, owner_sha256: String, - host_boot_micros: u64, + #[serde(flatten)] + host_boot: SelectionBoot, +} +/// Exactly one original qualifier; the output version is selected from it. +#[derive(Serialize)] +#[serde(untagged)] +enum SelectionBoot { + Legacy { host_boot_micros: u64 }, + Session { host_boot_uuid: host_boot::Session }, } struct Admitted<'a> { candidate: &'a Candidate, @@ -198,18 +206,26 @@ struct Admitted<'a> { } impl Admitted<'_> { fn selection(&self) -> Result { + let (version, host_boot) = match self.lease.host_boot().map_err(|_| refused())? { + owner::HostBoot::LegacyMicros(host_boot_micros) => { + (1, SelectionBoot::Legacy { host_boot_micros }) + } + owner::HostBoot::Session(host_boot_uuid) => { + (2, SelectionBoot::Session { host_boot_uuid }) + } + }; Ok(Selection { - version: 1, + version, kind: SelectionKind::NativeGraphRecoverySelection, run: self.run.into(), receipt: self.original.clone(), receipt_sha256: self.receipt_sha256.clone(), owner_sha256: self.lease.selected().fingerprint(), - host_boot_micros: self.lease.host_boot_micros().map_err(|_| refused())?, + host_boot, }) } } -/// Select only a dead version3 complete Ready publication, or its unchanged +/// Select only a dead boot-qualified complete Ready publication, or its unchanged /// committed recovery intent. No file is created or repaired, and no provider /// operation, private acquisition, signal or cleanup is performed. pub fn select(candidate: &Candidate, run: &str) -> Result { diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs index 9d67cca9a..45c835973 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs @@ -246,8 +246,8 @@ fn dead_complete_publication_selects_original_raw_hash_without_creating_authorit digest(&serde_json::to_vec(&selected.receipt).unwrap()) ); assert_eq!( - selected.host_boot_micros, - crate::provider::host_filesystem::host_boot_micros().unwrap() + serde_json::to_value(&selected).unwrap()["host_boot_uuid"], + serde_json::to_value(host_boot::read().unwrap()).unwrap() ); assert_eq!(fs::read(&state_path).unwrap(), original); assert_eq!(id(&state_path).unwrap(), state_id); @@ -328,7 +328,7 @@ fn selection_json_excludes_real_compiler_argv_environment_keys_values_and_owner_ .map(String::as_str) .collect::>(), [ - "host_boot_micros", + "host_boot_uuid", "kind", "owner_sha256", "receipt", @@ -377,11 +377,9 @@ fn dead_old_two_wrong_boot_and_unknown_publication_fields_refuse() { match case { "old-two" => { owner["version"] = json!(2); - owner.as_object_mut().unwrap().remove("host_boot_micros"); - } - "wrong-boot" => { - owner["host_boot_micros"] = json!(owner["host_boot_micros"].as_u64().unwrap() + 1) + owner.as_object_mut().unwrap().remove("host_boot_uuid"); } + "wrong-boot" => owner["host_boot_uuid"] = json!("00000000-0000-0000-0000-000000000001"), "unknown" => owner["cleanup_authority"] = json!(true), _ => unreachable!(), } @@ -390,6 +388,55 @@ fn dead_old_two_wrong_boot_and_unknown_publication_fields_refuse() { } } +#[test] +fn legacy_three_selection_remains_version_one_without_uuid_migration() { + let fixture = Fixture::new(); + fixture.dead(); + let path = fixture.owner_root().join("owner.json"); + let mut legacy: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + legacy["version"] = json!(3); + legacy.as_object_mut().unwrap().remove("host_boot_uuid"); + let micros = crate::provider::host_filesystem::host_boot_micros().unwrap(); + legacy["host_boot_micros"] = json!(micros); + fs::write(&path, legacy.to_string()).unwrap(); + let original = fs::read(&path).unwrap(); + let selected = serde_json::to_value(select(&fixture.candidate, RUN).unwrap()).unwrap(); + assert_eq!(selected["version"], 1); + assert_eq!(selected["host_boot_micros"], micros); + assert!(selected.get("host_boot_uuid").is_none()); + assert_eq!(fs::read(&path).unwrap(), original); + legacy["host_boot_micros"] = json!(micros + 1); + fs::write(&path, legacy.to_string()).unwrap(); + fixture.assert_refused_unchanged(); +} + +#[test] +fn recovery_session_is_rechecked_before_after_and_after_retirement() { + let fixture = Fixture::new(); + fixture.dead(); + let intent = fixture.intent(); + fixture.store(&intent); + let lease = owner::RecoveryLease::acquire( + &fixture.candidate, + RUN, + Some(intent.publication.clone()), + false, + false, + ) + .unwrap(); + lease.verify(false, false).unwrap(); + for changed in [Some("12345678-abcd-abcd-abcd-123456789abc"), None] { + let _drift = host_boot::test::Guard::set(changed); + assert!(lease.verify(false, false).is_err()); + assert!(lease.review().is_err()); + assert!(lease.archive(true).is_err()); + assert!(select(&fixture.candidate, RUN).is_err()); + assert!(fixture.owner_root().join("owner.json").exists()); + assert!(fixture.owner_root().join("control.sock").exists()); + } + lease.verify(false, false).unwrap(); +} + #[test] fn partial_failed_missing_id_or_pending_state_never_creates_an_intent() { for case in [ @@ -430,7 +477,7 @@ fn replaced_selected_owner_or_published_socket_and_lock_refuse_without_repair() let fixture = Fixture::new(); fixture.dead(); if name == "owner.json" { - // Publication3 carries no original owner-file inode. Its first + // Publication4 carries no original owner-file inode. Its first // private snapshot becomes the saved intent's later inode anchor. fixture.store(&fixture.intent()); } diff --git a/packages/runtime-core/src/provider/graph/native/foreground/tests.rs b/packages/runtime-core/src/provider/graph/native/foreground/tests.rs index d22fe9723..673a60f1a 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/tests.rs @@ -291,9 +291,15 @@ fn owner_codec_refuses_legacy_kind_unknown_fields_and_wrong_process_incarnation( ("/candidate", json!("/foreign")), ("/process/start_micros", json!(0)), ("/process/pid", json!(2_000_000)), - ("/host_boot_micros", json!(0)), - ("/host_boot_micros", json!(1)), - ("/host_boot_micros", json!(u64::MAX)), + ("/host_boot_uuid", json!(0)), + ( + "/host_boot_uuid", + json!("00000000-0000-0000-0000-000000000000"), + ), + ( + "/host_boot_uuid", + json!("00000000-0000-0000-0000-000000000001"), + ), ("/values", json!("private-canary")), ] { let fixture = Fixture::new(); @@ -318,24 +324,46 @@ fn owner_codec_refuses_legacy_kind_unknown_fields_and_wrong_process_incarnation( } #[test] -fn publication_three_binds_independent_host_boot_and_keeps_closed_live_version_two() { +fn publication_four_binds_session_and_preserves_closed_legacy_live_codecs() { let fixture = Fixture::new(); let review = fixture.receipt().review; let publication = owner::Publication::bind(&fixture.candidate, &review).unwrap(); let file = fixture.owner_root().join("owner.json"); let bytes = fs::read(&file).unwrap(); let qualified: Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(qualified["version"], 3); + assert_eq!(qualified["version"], 4); + assert_eq!( + qualified["host_boot_uuid"], + serde_json::to_value(host_boot::read().unwrap()).unwrap() + ); + assert!(qualified.get("host_boot_micros").is_none()); let boot = crate::provider::host_filesystem::host_boot_micros().unwrap(); - assert_eq!(qualified["host_boot_micros"], boot); - assert!(qualified["process"]["start_micros"].as_u64().unwrap() >= boot); + owner::Pin::load(&fixture.candidate, RUN).unwrap(); publication.verify().unwrap(); assert!(DirectGuard::acquire(&fixture.candidate, RUN).is_err()); + let mut legacy_three = qualified.clone(); + legacy_three["version"] = json!(3); + legacy_three + .as_object_mut() + .unwrap() + .remove("host_boot_uuid"); + legacy_three["host_boot_micros"] = json!(boot); + fs::write(&file, serde_json::to_vec(&legacy_three).unwrap()).unwrap(); + owner::Pin::load(&fixture.candidate, RUN) + .unwrap() + .connect() + .unwrap(); + assert!(publication.verify().is_err()); + // V3 remains strictly calendar-qualified rather than acquiring inferred UUID authority. + legacy_three["host_boot_micros"] = json!(boot + 1); + fs::write(&file, serde_json::to_vec(&legacy_three).unwrap()).unwrap(); + assert!(owner::Pin::load(&fixture.candidate, RUN).is_err()); + let mut old = qualified.clone(); old["version"] = json!(2); - old.as_object_mut().unwrap().remove("host_boot_micros"); + old.as_object_mut().unwrap().remove("host_boot_uuid"); fs::write(&file, serde_json::to_vec(&old).unwrap()).unwrap(); // Exact legacy live authentication remains available without inventing host boot. let legacy = owner::Pin::load(&fixture.candidate, RUN).unwrap(); @@ -357,6 +385,84 @@ fn publication_three_binds_independent_host_boot_and_keeps_closed_live_version_t assert!(!fixture.candidate.state_root.join("run/smolvm").exists()); } +#[test] +fn session_change_or_unavailable_reader_refuses_live_and_retired_fences() { + let fixture = Fixture::new(); + let review = fixture.receipt().review; + let first = "12345678-abcd-abcd-abcd-123456789abc"; + let _boot = host_boot::test::Guard::set(Some(first)); + let mut publication = owner::Publication::bind(&fixture.candidate, &review).unwrap(); + let pin = owner::Pin::load(&fixture.candidate, RUN).unwrap(); + pin.connect().unwrap(); + for changed in [Some("12345678-abcd-abcd-abcd-123456789abd"), None] { + let _drift = host_boot::test::Guard::set(changed); + assert!(pin.verify().is_err()); + assert!(pin.connect().is_err()); + assert!(publication.verify().is_err()); + assert!(publication.finish().is_err()); + assert!(fixture.owner_root().join("owner.json").exists()); + } + publication.finish().unwrap(); + pin.verify_retired().unwrap(); + let _drift = host_boot::test::Guard::set(None); + assert!(pin.verify_retired().is_err()); +} + +#[test] +fn session_codec_refuses_missing_mixed_downgraded_and_unknown_qualifiers() { + let fixture = Fixture::new(); + let publication = + owner::Publication::bind(&fixture.candidate, &fixture.receipt().review).unwrap(); + let file = fixture.owner_root().join("owner.json"); + let original: Value = serde_json::from_slice(&fs::read(&file).unwrap()).unwrap(); + for case in [ + "missing", + "null", + "uppercase", + "micros", + "v3", + "v2", + "future", + ] { + let mut bad = original.clone(); + match case { + "missing" => { + bad.as_object_mut().unwrap().remove("host_boot_uuid"); + } + "null" => bad["host_boot_uuid"] = Value::Null, + "uppercase" => bad["host_boot_uuid"] = json!("12345678-ABCD-ABCD-ABCD-123456789ABC"), + "micros" => bad["host_boot_micros"] = json!(1), + "v3" => bad["version"] = json!(3), + "v2" => bad["version"] = json!(2), + "future" => bad["version"] = json!(5), + _ => unreachable!(), + } + fs::write(&file, serde_json::to_vec(&bad).unwrap()).unwrap(); + assert!(owner::Pin::load(&fixture.candidate, RUN).is_err(), "{case}"); + assert!(publication.verify().is_err(), "{case}"); + } +} + +#[test] +fn uuid_drift_during_publication_and_peer_connect_refuses_final_fence() { + let first = "12345678-abcd-abcd-abcd-123456789abc"; + let changed = "12345678-abcd-abcd-abcd-123456789abd"; + let fixture = Fixture::new(); + { + let _drift = host_boot::test::Guard::sequence(&[Some(first), Some(changed)]); + assert!(owner::Publication::bind(&fixture.candidate, &fixture.receipt().review).is_err()); + assert!(fixture.owner_root().join("owner.json").exists()); + } + // Separate owned fixture keeps failed publication evidence intact. + let other = Fixture::new(); + let _boot = host_boot::test::Guard::set(Some(first)); + let _publication = owner::Publication::bind(&other.candidate, &other.receipt().review).unwrap(); + let pin = owner::Pin::load(&other.candidate, RUN).unwrap(); + let _drift = host_boot::test::Guard::sequence(&[Some(first), Some(changed)]); + assert!(pin.connect().is_err()); + assert!(other.owner_root().join("owner.json").exists()); +} + #[test] fn replaced_symlinked_hardlinked_and_public_owner_files_are_not_adopted() { for kind in ["replaced", "symlink", "hardlink", "public"] { diff --git a/src/backends/native-authored-recovery-protocol.ts b/src/backends/native-authored-recovery-protocol.ts index f0effc50a..0a53f3b0e 100644 --- a/src/backends/native-authored-recovery-protocol.ts +++ b/src/backends/native-authored-recovery-protocol.ts @@ -6,15 +6,19 @@ import { } from "./native-authored-graph-protocol.ts"; const HEX64 = /^[a-f0-9]{64}$/; -export type NativeAuthoredRecoverySelection = { - readonly version: 1; +const UUID = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/; +type RecoverySelectionFields = { readonly kind: "native-graph-recovery-selection"; readonly run: string; readonly receipt: NativeAuthoredReceipt; readonly receipt_sha256: string; readonly owner_sha256: string; - readonly host_boot_micros: number; }; +export type NativeAuthoredRecoverySelection = RecoverySelectionFields & + ( + | { readonly version: 1; readonly host_boot_micros: number } + | { readonly version: 2; readonly host_boot_uuid: string } + ); export type NativeAuthoredRecoveryResult = { readonly version: 1; readonly kind: "native-graph-live-owner-recovered"; @@ -61,6 +65,10 @@ export function parseNativeAuthoredRecoverySelection(opts: { }): NativeAuthoredRecoverySelection { const admitted = ready(opts.admitted); const value = opts.value; + const qualifier = + isRecord(value) && value.version === 2 + ? "host_boot_uuid" + : "host_boot_micros"; if ( !fields(value, [ "version", @@ -69,16 +77,13 @@ export function parseNativeAuthoredRecoverySelection(opts: { "receipt", "receipt_sha256", "owner_sha256", - "host_boot_micros", + qualifier, ]) || - value.version !== 1 || + (value.version !== 1 && value.version !== 2) || value.kind !== "native-graph-recovery-selection" || value.run !== admitted.review.provenance.run || !hash(value.receipt_sha256) || - !hash(value.owner_sha256) || - typeof value.host_boot_micros !== "number" || - !Number.isSafeInteger(value.host_boot_micros) || - value.host_boot_micros <= 0 + !hash(value.owner_sha256) ) { return refused(); } @@ -89,15 +94,31 @@ export function parseNativeAuthoredRecoverySelection(opts: { ) { return refused(); } - return { - version: 1, + const common: RecoverySelectionFields = { kind: "native-graph-recovery-selection", run: value.run, receipt, receipt_sha256: value.receipt_sha256, owner_sha256: value.owner_sha256, - host_boot_micros: value.host_boot_micros, }; + if (value.version === 1) { + if ( + typeof value.host_boot_micros !== "number" || + !Number.isSafeInteger(value.host_boot_micros) || + value.host_boot_micros <= 0 + ) { + return refused(); + } + return { ...common, version: 1, host_boot_micros: value.host_boot_micros }; + } + if ( + typeof value.host_boot_uuid !== "string" || + !UUID.test(value.host_boot_uuid) || + value.host_boot_uuid === "00000000-0000-0000-0000-000000000000" + ) { + return refused(); + } + return { ...common, version: 2, host_boot_uuid: value.host_boot_uuid }; } /** Matching Removed is a cleanup result, not frontend retirement authority. diff --git a/tests/native-authored-project-recovery.test.ts b/tests/native-authored-project-recovery.test.ts index ecbc12139..22fca634e 100644 --- a/tests/native-authored-project-recovery.test.ts +++ b/tests/native-authored-project-recovery.test.ts @@ -100,7 +100,7 @@ function removed() { ), }); } -async function fixture() { +async function fixture(bootSession = false) { const root = await realpath( await mkdtemp(join(tmpdir(), "native-frontend-recovery-")) ); @@ -153,13 +153,15 @@ async function fixture() { const calls: string[][] = []; let cleanupCalls = 0; const selection = { - version: 1, + version: bootSession ? 2 : 1, kind: "native-graph-recovery-selection", run: "a".repeat(32), receipt: receipt(), receipt_sha256: "4".repeat(64), owner_sha256: "5".repeat(64), - host_boot_micros: 1_791_000_000_000_000, + ...(bootSession + ? { host_boot_uuid: "12345678-abcd-abcd-abcd-123456789abc" } + : { host_boot_micros: 1_791_000_000_000_000 }), }; const result = { version: 1, @@ -297,6 +299,21 @@ test("dead frontend recovery binds the stored run, holds startup, retires only a ).toBe(1); }); +test("frontend recovery preserves UUID selection through complete retry without changing Ready or control versions", async () => { + const current = await fixture(true); + await current.kill(); + await recoverNativeAuthoredProject(current.options); + const saved = JSON.parse(await Bun.file(current.paths.intent).text()); + expect(saved.record.native.version).toBe(2); + expect(saved.record.native.host_boot_uuid).toBe( + "12345678-abcd-abcd-abcd-123456789abc" + ); + expect(saved.record.native).not.toHaveProperty("host_boot_micros"); + expect(saved.record.native.receipt.version).toBe(2); + await recoverNativeAuthoredProject(current.options); + expect(current.cleanupCalls()).toBe(1); +}); + test("recovery refuses foreign Removed membership or nonnull current observations and retains frontend authority", async () => { for (const attack of ["id", "observation"]) { const current = await fixture(); diff --git a/tests/native-authored-recovery-protocol.test.ts b/tests/native-authored-recovery-protocol.test.ts index 21f0c5937..050bcd19e 100644 --- a/tests/native-authored-recovery-protocol.test.ts +++ b/tests/native-authored-recovery-protocol.test.ts @@ -68,6 +68,14 @@ function selection() { host_boot_micros: 1_791_000_000_000_000, }; } +function sessionSelection() { + const { host_boot_micros: _legacy, ...common } = selection(); + return { + ...common, + version: 2, + host_boot_uuid: "12345678-abcd-abcd-abcd-123456789abc", + }; +} function result() { const removed = receipt(); removed.phase = "removed"; @@ -99,6 +107,47 @@ test("native recovery copies original selectors and binds exact Removed to durab .phase ).toBe("removed"); }); +test("session selection is closed version two and preserves original qualifier without migration", () => { + const admitted = parseNativeAuthoredReceipt(receipt()); + const expected = parseNativeAuthoredRecoverySelection({ + value: sessionSelection(), + admitted, + }); + expect(expected.version).toBe(2); + expect(expected).toHaveProperty( + "host_boot_uuid", + sessionSelection().host_boot_uuid + ); + expect(expected).not.toHaveProperty("host_boot_micros"); + expect( + parseNativeAuthoredRecoveryResult({ value: result(), expected }).version + ).toBe(1); + const canary = "private-boot-qualifier-canary"; + const { host_boot_uuid: _missing, ...missing } = sessionSelection(); + for (const value of [ + missing, + { ...sessionSelection(), version: 1 }, + { ...sessionSelection(), version: 3 }, + { ...sessionSelection(), host_boot_micros: 1 }, + { ...selection(), host_boot_uuid: sessionSelection().host_boot_uuid }, + ...[ + null, + 1, + canary, + "12345678-ABCD-ABCD-ABCD-123456789ABC", + "00000000-0000-0000-0000-000000000000", + "12345678-abcd-abcd-abcd-123456789abg", + ].map((host_boot_uuid) => ({ ...sessionSelection(), host_boot_uuid })), + ]) { + try { + parseNativeAuthoredRecoverySelection({ value, admitted }); + throw new Error("fixture admitted forbidden qualifier"); + } catch (error) { + expect(String(error)).toContain("invalid"); + expect(String(error)).not.toContain(canary); + } + } +}); test("native recovery selection refuses inherited unknown private or invalid boot/hash fields", () => { const admitted = parseNativeAuthoredReceipt(receipt()); const canary = "private-recovery-wire-canary"; From 73c7483131306efecfe8d7e65cbf25c9b5d1fff6 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 20:17:52 -0400 Subject: [PATCH 45/47] test(native): fence final boot observation and codec admission --- .../graph/native/foreground/recovery/tests.rs | 54 +++++++++++++++++-- .../provider/graph/native/foreground/tests.rs | 6 +-- .../native-authored-recovery-protocol.ts | 2 +- .../native-authored-recovery-protocol.test.ts | 30 +++++++++++ 4 files changed, 84 insertions(+), 8 deletions(-) diff --git a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs index 45c835973..8376a8034 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/recovery/tests.rs @@ -245,9 +245,10 @@ fn dead_complete_publication_selects_original_raw_hash_without_creating_authorit selected.receipt_sha256, digest(&serde_json::to_vec(&selected.receipt).unwrap()) ); - assert_eq!( - serde_json::to_value(&selected).unwrap()["host_boot_uuid"], - serde_json::to_value(host_boot::read().unwrap()).unwrap() + assert!( + serde_json::to_value(&selected).unwrap()["host_boot_uuid"] + == serde_json::to_value(host_boot::read().unwrap()).unwrap(), + "Selection must retain the observed boot qualifier" ); assert_eq!(fs::read(&state_path).unwrap(), original); assert_eq!(id(&state_path).unwrap(), state_id); @@ -411,7 +412,7 @@ fn legacy_three_selection_remains_version_one_without_uuid_migration() { } #[test] -fn recovery_session_is_rechecked_before_after_and_after_retirement() { +fn recovery_session_entry_refusal_and_restoration_preserve_exact_evidence() { let fixture = Fixture::new(); fixture.dead(); let intent = fixture.intent(); @@ -437,6 +438,51 @@ fn recovery_session_is_rechecked_before_after_and_after_retirement() { lease.verify(false, false).unwrap(); } +#[test] +fn recovery_session_final_observation_refuses_drift_without_changing_original_evidence() { + let fixture = Fixture::new(); + fixture.dead(); + let intent = fixture.intent(); + fixture.store(&intent); + let lease = owner::RecoveryLease::acquire( + &fixture.candidate, + RUN, + Some(intent.publication.clone()), + false, + false, + ) + .unwrap(); + let current = serde_json::to_value(host_boot::read().unwrap()) + .unwrap() + .as_str() + .unwrap() + .to_owned(); + let mut changed = current.clone(); + changed.replace_range(0..1, if current.starts_with('f') { "e" } else { "f" }); + let owner = fixture.owner_root().join("owner.json"); + let socket = fixture.owner_root().join("control.sock"); + let journal = fixture.journal_root().join("state.json"); + let saved_intent = fixture.journal_root().join(FILE); + let paths = [&owner, &socket, &journal, &saved_intent]; + let identities: Vec<_> = paths.iter().map(|path| id(path).unwrap()).collect(); + let bytes = [&owner, &journal, &saved_intent].map(|path| fs::read(path).unwrap()); + for last in [Some(changed.as_str()), None] { + let _drift = host_boot::test::Guard::sequence(&[Some(¤t), last]); + // First record read admits the original boot; final read must refuse. + assert!(lease.verify(false, false).is_err()); + for (path, expected) in paths.iter().zip(&identities) { + assert_eq!(id(path).unwrap(), *expected); + } + for (path, expected) in [&owner, &journal, &saved_intent].into_iter().zip(&bytes) { + assert!( + fs::read(path).unwrap() == *expected, + "Original evidence must remain unchanged" + ); + } + } + lease.verify(false, false).unwrap(); +} + #[test] fn partial_failed_missing_id_or_pending_state_never_creates_an_intent() { for case in [ diff --git a/packages/runtime-core/src/provider/graph/native/foreground/tests.rs b/packages/runtime-core/src/provider/graph/native/foreground/tests.rs index 673a60f1a..709c7e767 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/tests.rs @@ -332,9 +332,9 @@ fn publication_four_binds_session_and_preserves_closed_legacy_live_codecs() { let bytes = fs::read(&file).unwrap(); let qualified: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(qualified["version"], 4); - assert_eq!( - qualified["host_boot_uuid"], - serde_json::to_value(host_boot::read().unwrap()).unwrap() + assert!( + qualified["host_boot_uuid"] == serde_json::to_value(host_boot::read().unwrap()).unwrap(), + "Publication must retain the observed boot qualifier" ); assert!(qualified.get("host_boot_micros").is_none()); let boot = crate::provider::host_filesystem::host_boot_micros().unwrap(); diff --git a/src/backends/native-authored-recovery-protocol.ts b/src/backends/native-authored-recovery-protocol.ts index 0a53f3b0e..3af0cd799 100644 --- a/src/backends/native-authored-recovery-protocol.ts +++ b/src/backends/native-authored-recovery-protocol.ts @@ -66,7 +66,7 @@ export function parseNativeAuthoredRecoverySelection(opts: { const admitted = ready(opts.admitted); const value = opts.value; const qualifier = - isRecord(value) && value.version === 2 + isRecord(value) && Object.hasOwn(value, "host_boot_uuid") ? "host_boot_uuid" : "host_boot_micros"; if ( diff --git a/tests/native-authored-recovery-protocol.test.ts b/tests/native-authored-recovery-protocol.test.ts index 050bcd19e..92c742f30 100644 --- a/tests/native-authored-recovery-protocol.test.ts +++ b/tests/native-authored-recovery-protocol.test.ts @@ -148,6 +148,36 @@ test("session selection is closed version two and preserves original qualifier w } } }); +test("selector dispatch refuses missing own fields before reading version getters", () => { + const admitted = parseNativeAuthoredReceipt(receipt()); + let calls = 0; + const prototype = { + get version() { + calls++; + return 2; + }, + }; + const ownPartial = Object.create(null); + Object.defineProperties(ownPartial, { + version: { + enumerable: true, + get() { + calls++; + return 2; + }, + }, + host_boot_uuid: { + enumerable: true, + value: sessionSelection().host_boot_uuid, + }, + }); + for (const value of [Object.create(prototype), ownPartial]) { + expect(() => + parseNativeAuthoredRecoverySelection({ value, admitted }) + ).toThrow("invalid"); + expect(calls).toBe(0); + } +}); test("native recovery selection refuses inherited unknown private or invalid boot/hash fields", () => { const admitted = parseNativeAuthoredReceipt(receipt()); const canary = "private-recovery-wire-canary"; From bbb808702e501f7b9640f046fecb1cd38542d1c1 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 20:20:19 -0400 Subject: [PATCH 46/47] fix(native): require exact boot UUID wire length --- .../runtime-core/src/provider/graph/native/foreground/tests.rs | 1 + src/backends/native-authored-recovery-protocol.ts | 1 + tests/native-authored-recovery-protocol.test.ts | 2 ++ 3 files changed, 4 insertions(+) diff --git a/packages/runtime-core/src/provider/graph/native/foreground/tests.rs b/packages/runtime-core/src/provider/graph/native/foreground/tests.rs index 709c7e767..6f4222abd 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/tests.rs @@ -290,6 +290,7 @@ fn owner_codec_refuses_legacy_kind_unknown_fields_and_wrong_process_incarnation( ("/kind", json!("compose")), ("/candidate", json!("/foreign")), ("/process/start_micros", json!(0)), + ("/process/start_micros", json!(1)), ("/process/pid", json!(2_000_000)), ("/host_boot_uuid", json!(0)), ( diff --git a/src/backends/native-authored-recovery-protocol.ts b/src/backends/native-authored-recovery-protocol.ts index 3af0cd799..c48cd4cb9 100644 --- a/src/backends/native-authored-recovery-protocol.ts +++ b/src/backends/native-authored-recovery-protocol.ts @@ -113,6 +113,7 @@ export function parseNativeAuthoredRecoverySelection(opts: { } if ( typeof value.host_boot_uuid !== "string" || + value.host_boot_uuid.length !== 36 || !UUID.test(value.host_boot_uuid) || value.host_boot_uuid === "00000000-0000-0000-0000-000000000000" ) { diff --git a/tests/native-authored-recovery-protocol.test.ts b/tests/native-authored-recovery-protocol.test.ts index 92c742f30..8bc208514 100644 --- a/tests/native-authored-recovery-protocol.test.ts +++ b/tests/native-authored-recovery-protocol.test.ts @@ -137,6 +137,8 @@ test("session selection is closed version two and preserves original qualifier w "12345678-ABCD-ABCD-ABCD-123456789ABC", "00000000-0000-0000-0000-000000000000", "12345678-abcd-abcd-abcd-123456789abg", + "12345678-abcd-abcd-abcd-123456789abc\n", + "12345678-abcd-abcd-abcd-123456789abc ", ].map((host_boot_uuid) => ({ ...sessionSelection(), host_boot_uuid })), ]) { try { From 87425d1b41c4eb6b75af638df11f5cc980fbbd95 Mon Sep 17 00:00:00 2001 From: hack-cli-tests Date: Thu, 8 Oct 2026 20:34:14 -0400 Subject: [PATCH 47/47] fix: preserve omitted profiles for native recovery --- docs/reference/native-authored-commands.md | 2 + src/commands/project.ts | 5 ++- .../native-authored-command-dispatch.test.ts | 44 +++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/docs/reference/native-authored-commands.md b/docs/reference/native-authored-commands.md index 71b378eed..39a940976 100644 --- a/docs/reference/native-authored-commands.md +++ b/docs/reference/native-authored-commands.md @@ -44,6 +44,8 @@ Live owners, older version 2 dead publications, partial startup, pending writes, changed files and rebooted guests refuse. Env/profile changes, service subsets and `--json` are unsupported for this operation. It neither recompiles authored input nor acquires managed values or starts a provider. +Omit `--profile` to recover the stored generation; explicit empty or named profile +overrides both refuse. Recovery commits a distinct private intent before calling the Rust cleanup owner. The original raw selectors and resource inventory remain fixed through retries; diff --git a/src/commands/project.ts b/src/commands/project.ts index 8da6fe31b..b37d715a6 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -6705,7 +6705,10 @@ async function handleDown({ project: args.options.project, operation: "down", instance: args.options.branch, - profiles: parseCsvList(args.options.profile), + profiles: + args.options.profile === undefined + ? undefined + : parseCsvList(args.options.profile), overlay: args.options.env, json: args.options.json, recover: args.options.recoverNativeCompose, diff --git a/tests/native-authored-command-dispatch.test.ts b/tests/native-authored-command-dispatch.test.ts index a676b2d29..9a2ac6f94 100644 --- a/tests/native-authored-command-dispatch.test.ts +++ b/tests/native-authored-command-dispatch.test.ts @@ -154,6 +154,50 @@ test("source CLI mixed authored families refuse before native dispatch", async ( }); const macTest = process.platform === "darwin" ? test : test.skip; +macTest.each([ + { name: "omitted", args: [], code: "E_LIFECYCLE_FAILED" }, + { + name: "explicit-empty", + args: ["--profile", ""], + code: "E_NATIVE_PROJECT_UNSUPPORTED", + }, + { + name: "explicit-named", + args: ["--profile", "dev"], + code: "E_NATIVE_PROJECT_UNSUPPORTED", + }, +])( + "source CLI recovery keeps $name profile selection without input or runtime work", + async ({ args, code }) => { + const selected = await fixture(); + await Bun.write( + join(selected.root, ".hack/hack.env.default.yaml"), + `values: [${PRIVATE}` + ); + const value = await invoke({ + selected, + args: ["down", "--recover", ...args], + backend: "native", + logger: "console", + }); + expect(value.code).toBe(1); + expect(value.stdout + value.stderr).toMatch( + new RegExp(`(?:^|\\n)ERROR: ${code}\\b`) + ); + expect(value.stdout + value.stderr).not.toContain( + code === "E_LIFECYCLE_FAILED" + ? "E_NATIVE_PROJECT_UNSUPPORTED" + : "E_LIFECYCLE_FAILED" + ); + expect( + await Bun.file(join(selected.root, "compiler-called")).exists() + ).toBe(false); + expect(await readdir(join(selected.root, ".hack"))).toEqual([ + "hack.env.default.yaml", + "hack.project.json", + ]); + } +); const unsupportedFiles = [ { name: "configs-empty", files: { configs: {} } }, { name: "secrets-empty", files: { secrets: {} } },