From 4afb2fa24d9b1957324336a331c83719d88438e4 Mon Sep 17 00:00:00 2001 From: cxymds Date: Sun, 13 Sep 2026 22:39:59 +0800 Subject: [PATCH] fix(heal): preserve incomplete and unknown cluster status --- crates/cli/src/commands/admin/heal.rs | 91 ++++++++- crates/cli/tests/admin_heal_status.rs | 269 ++++++++++++++++++++++++++ crates/core/src/admin/cluster.rs | 21 ++ crates/core/src/admin/mod.rs | 12 +- crates/s3/src/admin.rs | 42 +++- schemas/output_v2.json | 32 ++- 6 files changed, 448 insertions(+), 19 deletions(-) create mode 100644 crates/cli/tests/admin_heal_status.rs diff --git a/crates/cli/src/commands/admin/heal.rs b/crates/cli/src/commands/admin/heal.rs index a69da4b5..7581cfa8 100644 --- a/crates/cli/src/commands/admin/heal.rs +++ b/crates/cli/src/commands/admin/heal.rs @@ -9,7 +9,8 @@ use super::get_admin_client; use crate::exit_code::ExitCode; use crate::output::Formatter; use rc_core::admin::{ - AdminApi, HealRuntimeState, HealScanMode, HealStartRequest, HealStatus, HealTaskRequest, + AdminApi, BackgroundHealCoverage, HealRuntimeState, HealScanMode, HealStartRequest, HealStatus, + HealTaskRequest, }; const HEAL_STOP_SUCCESS_MESSAGE: &str = "Heal operation stopped successfully"; @@ -104,6 +105,10 @@ struct HealStatusOutput { #[serde(skip_serializing_if = "Option::is_none")] state: Option, #[serde(skip_serializing_if = "Option::is_none")] + cluster_status_complete: Option, + #[serde(skip_serializing_if = "Option::is_none")] + coverage: Option, + #[serde(skip_serializing_if = "Option::is_none")] summary: Option, #[serde(skip_serializing_if = "Option::is_none")] detail: Option, @@ -131,6 +136,8 @@ impl From<&HealStatus> for HealStatusOutput { heal_id: status.heal_id.clone(), healing: status.healing, state: status.state, + cluster_status_complete: status.cluster_status_complete, + coverage: status.coverage.clone(), summary: status.summary.clone(), detail: status.detail.clone(), bucket: status.bucket.clone(), @@ -153,6 +160,8 @@ impl From<&HealStatus> for HealStatusOutput { fn has_heal_status_details(status: &HealStatus) -> bool { status.healing || status.state.is_some() + || status.cluster_status_complete.is_some() + || status.coverage.is_some() || !status.heal_id.is_empty() || status.summary.is_some() || status.detail.is_some() @@ -210,13 +219,29 @@ enum HealStatusIndicator { Date(&'static str), } +fn cluster_heal_status_is_incomplete(status: &HealStatus) -> bool { + status.state == Some(HealRuntimeState::Degraded) + || status.cluster_status_complete == Some(false) + || status.coverage.as_ref().is_some_and(|coverage| { + coverage.unknown.is_some_and(|count| count > 0) + || !coverage.reasons.is_empty() + || matches!( + (coverage.responded, coverage.expected), + (Some(responded), Some(expected)) if responded < expected + ) + }) +} + fn heal_status_indicator(status: &HealStatus) -> HealStatusIndicator { match status.state { + Some(HealRuntimeState::Active) => HealStatusIndicator::Progress("In Progress"), + Some(HealRuntimeState::Degraded) => HealStatusIndicator::Date("Degraded"), + Some(HealRuntimeState::Unknown) => HealStatusIndicator::Date("Unknown"), + _ if cluster_heal_status_is_incomplete(status) => HealStatusIndicator::Date("Unknown"), Some(HealRuntimeState::Disabled) => HealStatusIndicator::Date("Disabled"), Some(HealRuntimeState::Uninitialized) => HealStatusIndicator::Date("Uninitialized"), - Some(HealRuntimeState::Active) => HealStatusIndicator::Progress("In Progress"), Some(HealRuntimeState::Idle) => HealStatusIndicator::Date("Idle"), - Some(HealRuntimeState::Unknown) | None => match status.summary.as_deref() { + None => match status.summary.as_deref() { Some("running") => HealStatusIndicator::Progress("In Progress"), Some("finished") => HealStatusIndicator::Progress("Finished"), Some("stopped") => HealStatusIndicator::Date("Stopped"), @@ -287,6 +312,42 @@ fn print_heal_status(status: &HealStatus, formatter: &Formatter) { )); formatter.println(""); + let incomplete = cluster_heal_status_is_incomplete(status); + let unknown = status.state == Some(HealRuntimeState::Unknown); + if incomplete { + formatter + .warning("Cluster heal status is incomplete; unavailable peers may still be healing."); + } else if unknown { + formatter.warning("Heal runtime state is unknown; cluster idleness cannot be confirmed."); + } + + // Token responses describe one task, not the coverage of a cluster snapshot. + if status.summary.is_none() && status.heal_id.is_empty() { + let completeness = if incomplete { + "Incomplete" + } else if status.cluster_status_complete == Some(true) { + "Complete" + } else { + "Unknown (not reported by server)" + }; + formatter.println(&format!(" Cluster status: {completeness}")); + } + if let Some(coverage) = &status.coverage { + let count = |value: Option| value.map_or_else(|| "?".to_string(), |n| n.to_string()); + formatter.println(&format!( + " Node coverage: {}/{} responded, {} unknown", + count(coverage.responded), + count(coverage.expected), + count(coverage.unknown), + )); + if !coverage.reasons.is_empty() { + formatter.println(&format!( + " Reasons: {}", + formatter.sanitize_text(&coverage.reasons.join(", ")) + )); + } + } + if !status.heal_id.is_empty() { formatter.println(&format!(" Heal ID: {}", status.heal_id)); } @@ -336,6 +397,12 @@ fn print_heal_status(status: &HealStatus, formatter: &Formatter) { if let Some(ref last_update) = status.last_update { formatter.println(&format!(" Last Update: {}", last_update)); } + } else if incomplete || unknown { + formatter.println(" Cluster idleness cannot be confirmed."); + } else if status.state == Some(HealRuntimeState::Disabled) { + formatter.println(" Heal service is disabled."); + } else if status.state == Some(HealRuntimeState::Uninitialized) { + formatter.println(" Heal service is not initialized."); } else { formatter.println(" No active heal operation."); } @@ -714,4 +781,22 @@ mod tests { HealStatusIndicator::Date("Disabled") )); } + + #[test] + fn test_explicit_unknown_overrides_legacy_summary_and_healing() { + for summary in [None, Some("running"), Some("finished"), Some("stopped")] { + for healing in [false, true] { + let status = HealStatus { + state: Some(HealRuntimeState::Unknown), + summary: summary.map(str::to_string), + healing, + ..Default::default() + }; + assert!(matches!( + heal_status_indicator(&status), + HealStatusIndicator::Date("Unknown") + )); + } + } + } } diff --git a/crates/cli/tests/admin_heal_status.rs b/crates/cli/tests/admin_heal_status.rs new file mode 100644 index 00000000..cdbdded7 --- /dev/null +++ b/crates/cli/tests/admin_heal_status.rs @@ -0,0 +1,269 @@ +#![cfg(not(windows))] + +mod admin_support; + +use std::process::{Command, Output}; +use std::time::Duration; + +use admin_support::{rc_binary, rc_host_alias, start_admin_test_server}; +use serde_json::Value; + +fn run_status(response: &'static str, json: bool, token: bool) -> Output { + let config_dir = tempfile::tempdir().expect("create isolated config"); + let (endpoint, receiver, handle) = start_admin_test_server(response); + let mut command = Command::new(rc_binary()); + command.arg("--no-color"); + if json { + command.arg("--json"); + } + command.args(["admin", "heal", "status", "myalias"]); + if token { + command.args(["--client-token", "25080000-0000-4000-8000-000000000001"]); + } + let output = command + .env("RC_CONFIG_DIR", config_dir.path()) + .env("RC_HOST_myalias", rc_host_alias(&endpoint)) + .env("NO_PROXY", "127.0.0.1,localhost") + .env("no_proxy", "127.0.0.1,localhost") + .output() + .expect("run heal status"); + assert_eq!( + output.status.code(), + Some(0), + "status lookup must retain its exit contract: {}", + String::from_utf8_lossy(&output.stderr) + ); + let request = receiver + .recv_timeout(Duration::from_secs(5)) + .expect("capture status request"); + assert_eq!(request.method, "POST"); + assert_eq!( + request.target, + if token { + "/rustfs/admin/v3/heal/?clientToken=25080000-0000-4000-8000-000000000001" + } else { + "/rustfs/admin/v3/background-heal/status" + } + ); + handle.join().expect("status server finished"); + output +} + +fn status_outputs(response: &'static str, token: bool) -> (Value, String, String) { + let json_output = run_status(response, true, token); + assert!(json_output.stderr.is_empty()); + let json: Value = serde_json::from_slice(&json_output.stdout).expect("decode status output"); + let schema: Value = serde_json::from_str(include_str!("../../../schemas/output_v2.json")) + .expect("decode output schema"); + let validator = jsonschema::validator_for(&serde_json::json!({ + "$ref": "#/definitions/healStatus", + "definitions": schema["definitions"], + })) + .expect("compile heal status schema"); + assert!(validator.is_valid(&json), "status violates schema: {json}"); + let text_output = run_status(response, false, token); + ( + json, + String::from_utf8(text_output.stdout).expect("status text"), + String::from_utf8(text_output.stderr).expect("status warnings"), + ) +} + +#[test] +fn degraded_status_preserves_partial_coverage_in_json_and_text() { + let (json, text, warning) = status_outputs( + r#"{"state":"degraded","clusterStatusComplete":false,"coverage":{"expected":4,"responded":3,"unknown":1,"reasons":["peer_status_unavailable"]}}"#, + false, + ); + assert_eq!(json["state"], "degraded"); + assert_eq!(json["healing"], false); + assert_eq!(json["clusterStatusComplete"], false); + assert_eq!(json["coverage"]["expected"], 4); + assert_eq!(json["coverage"]["responded"], 3); + assert_eq!(json["coverage"]["unknown"], 1); + assert_eq!(json["coverage"]["reasons"][0], "peer_status_unavailable"); + assert!(text.contains("Heal Status: Degraded")); + assert!(text.contains("3/4 responded")); + assert!(text.contains("1 unknown")); + assert!(text.contains("peer_status_unavailable")); + assert!(warning.contains("incomplete")); + assert!(!text.contains("Idle")); + assert!(!text.contains("No active heal operation")); +} + +#[test] +fn explicit_unknown_and_future_states_never_become_idle() { + for response in [ + r#"{"state":"unknown"}"#, + r#"{"state":"future_state"}"#, + r#"{"state":"future_state","currentScanMode":2,"bitrotStartTime":"2026-09-13T13:00:00Z"}"#, + ] { + let (json, text, warning) = status_outputs(response, false); + assert_eq!(json["state"], "unknown"); + assert_eq!(json["healing"], false); + assert!(text.contains("Heal Status: Unknown")); + assert!(warning.contains("unknown")); + assert!(!text.contains("Idle")); + assert!(!text.contains("Finished")); + assert!(!text.contains("No active heal operation")); + assert!(json.get("clusterStatusComplete").is_none()); + assert!(json.get("coverage").is_none()); + } +} + +#[test] +fn unknown_runtime_retains_observed_tasks_without_claiming_a_known_state() { + let (json, text, warning) = status_outputs( + r#"{"state":"future_state","healOperations":{"queueLength":2,"activeTasks":1}}"#, + false, + ); + assert_eq!(json["state"], "unknown"); + assert_eq!(json["healing"], true); + assert_eq!(json["healQueueLength"], 2); + assert_eq!(json["healActiveTasks"], 1); + assert!(text.contains("Heal Status: Unknown")); + assert!(text.contains("2 queued, 1 active")); + assert!(warning.contains("unknown")); + assert!(!text.contains("No active heal operation")); +} + +#[test] +fn active_partial_status_keeps_activity_and_incomplete_warning() { + let (json, text, warning) = status_outputs( + r#"{"state":"active","clusterStatusComplete":false,"coverage":{"expected":4,"responded":3,"unknown":1,"reasons":["future_reason"]}}"#, + false, + ); + assert_eq!(json["state"], "active"); + assert_eq!(json["healing"], true); + assert_eq!(json["clusterStatusComplete"], false); + assert_eq!(json["coverage"]["reasons"][0], "future_reason"); + assert!(text.contains("Heal Status: In Progress")); + assert!(text.contains("3/4 responded")); + assert!(text.contains("future_reason")); + assert!(warning.contains("incomplete")); +} + +#[test] +fn complete_idle_and_runtime_unavailability_remain_distinct() { + for (response, state, label) in [ + ( + r#"{"state":"idle","clusterStatusComplete":true,"coverage":{"expected":4,"responded":4,"unknown":0,"reasons":[]}}"#, + "idle", + "Idle", + ), + ( + r#"{"state":"disabled","clusterStatusComplete":true,"coverage":{"expected":4,"responded":4,"unknown":0,"reasons":[]}}"#, + "disabled", + "Disabled", + ), + ( + r#"{"state":"uninitialized","clusterStatusComplete":true,"coverage":{"expected":4,"responded":4,"unknown":0,"reasons":[]}}"#, + "uninitialized", + "Uninitialized", + ), + ] { + let (json, text, warning) = status_outputs(response, false); + assert_eq!(json["state"], state); + assert_eq!(json["healing"], false); + assert_eq!(json["clusterStatusComplete"], true); + assert_eq!(json["coverage"]["unknown"], 0); + assert!(text.contains(&format!("Heal Status: {label}"))); + assert!(text.contains("4/4 responded")); + assert!(warning.is_empty()); + if state != "idle" { + assert!(!text.contains("No active heal operation")); + } + } +} + +#[test] +fn missing_coverage_counts_stay_unknown_and_legacy_state_still_works() { + let (json, text, warning) = status_outputs(r#"{"state":"degraded","coverage":{}}"#, false); + assert!(json.get("clusterStatusComplete").is_none()); + for count in ["expected", "responded", "unknown"] { + assert!(json["coverage"].get(count).is_none()); + } + assert!(text.contains("?/? responded")); + assert!(warning.contains("incomplete")); + + for (response, healing, label) in [ + (r#"{}"#, false, "Idle"), + (r#"{"healQueueLength":1}"#, true, "In Progress"), + (r#"{"currentScanMode":2}"#, true, "In Progress"), + ] { + let (json, text, _) = status_outputs(response, false); + assert!(json.get("state").is_none()); + assert!(json.get("clusterStatusComplete").is_none()); + assert!(json.get("coverage").is_none()); + assert_eq!(json["healing"], healing); + assert!(text.contains(&format!("Heal Status: {label}"))); + assert!(text.contains("Cluster status: Unknown")); + } +} + +#[test] +fn coverage_reasons_preserve_json_but_cannot_inject_terminal_lines() { + let (json, text, warning) = status_outputs( + r#"{"state":"degraded","coverage":{"reasons":["future\nforged_line\u001b[2J"]}}"#, + false, + ); + assert_eq!( + json["coverage"]["reasons"][0], + "future\nforged_line\u{1b}[2J" + ); + assert!(!text.contains('\u{1b}')); + assert!(!text.lines().any(|line| line.starts_with("forged_line"))); + assert!(warning.contains("incomplete")); +} + +#[test] +fn explicit_incompleteness_cannot_fall_back_to_legacy_idle() { + for response in [ + r#"{"clusterStatusComplete":false}"#, + r#"{"state":"idle","clusterStatusComplete":false}"#, + r#"{"state":"idle","coverage":{"expected":4,"responded":3}}"#, + r#"{"state":"idle","coverage":{"unknown":1}}"#, + r#"{"state":"idle","coverage":{"reasons":["peer_status_unavailable"]}}"#, + ] { + let (_, text, warning) = status_outputs(response, false); + assert!(text.contains("Heal Status: Unknown")); + assert!(warning.contains("incomplete")); + assert!(!text.contains("No active heal operation")); + } +} + +#[test] +fn token_status_preserves_legacy_summaries_and_terminal_error_detail() { + for (response, summary, label, healing) in [ + (r#"{"summary":"running"}"#, "running", "In Progress", true), + (r#"{"summary":"finished"}"#, "finished", "Finished", false), + (r#"{"summary":"stopped"}"#, "stopped", "Stopped", false), + (r#"{"summary":"notFound"}"#, "notFound", "Not Found", false), + ( + r#"{"summary":"stopped","detail":"heal traversal completed with errors: 1 failed objects","progress":{"objectsScanned":1,"objectsFailed":1},"outcome":{"execution":{"state":"completed_with_errors"}}}"#, + "stopped", + "Stopped", + false, + ), + ] { + let (json, text, warning) = status_outputs(response, true); + assert_eq!(json["summary"], summary); + assert_eq!(json["healing"], healing); + assert!(json.get("state").is_none()); + assert!(json.get("clusterStatusComplete").is_none()); + assert!(json.get("coverage").is_none()); + assert!(text.contains(&format!("Heal Status: {label}"))); + assert!(!text.contains("Cluster status:")); + assert!(warning.is_empty()); + if response.contains("completed_with_errors") { + assert_eq!(json["itemsFailed"], 1); + assert!( + json["detail"] + .as_str() + .unwrap() + .contains("completed with errors") + ); + assert!(text.contains("completed with errors")); + } + } +} diff --git a/crates/core/src/admin/cluster.rs b/crates/core/src/admin/cluster.rs index 844310b6..3a5c93b5 100644 --- a/crates/core/src/admin/cluster.rs +++ b/crates/core/src/admin/cluster.rs @@ -463,10 +463,24 @@ pub enum HealRuntimeState { Uninitialized, Idle, Active, + Degraded, #[serde(other)] Unknown, } +/// Node coverage of a background heal snapshot; missing counts are unknown. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct BackgroundHealCoverage { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expected: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub responded: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub unknown: Option, + #[serde(default)] + pub reasons: Vec, +} + /// Request to start a heal operation #[derive(Debug, Clone, Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase")] @@ -599,6 +613,13 @@ pub struct HealStatus { #[serde(default, skip_serializing_if = "Option::is_none")] pub state: Option, + /// Absent on legacy servers and token-scoped task responses. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cluster_status_complete: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub coverage: Option, + /// Task summary for token-scoped manual heal status #[serde(default, skip_serializing_if = "Option::is_none")] pub summary: Option, diff --git a/crates/core/src/admin/mod.rs b/crates/core/src/admin/mod.rs index 92b0086c..34d47f53 100644 --- a/crates/core/src/admin/mod.rs +++ b/crates/core/src/admin/mod.rs @@ -47,12 +47,12 @@ pub use capabilities::{ RuntimeCapabilityStatus, }; pub use cluster::{ - BackendInfo, BackendType, BucketsInfo, ClusterInfo, DecommissionPoolStatus, DecommissionStatus, - DiskInfo, HealDriveInfo, HealDriveInfos, HealResultItem, HealRuntimeState, HealScanMode, - HealStartRequest, HealStatus, HealTaskRequest, HealingDiskInfo, MemStats, ObjectsInfo, - PoolDecommissionInfo, PoolErasureSetInfo, PoolStatus, PoolTarget, RebalanceCleanupWarnings, - RebalancePoolProgress, RebalancePoolStatus, RebalanceStartResult, RebalanceStatus, ServerInfo, - UsageInfo, + BackendInfo, BackendType, BackgroundHealCoverage, BucketsInfo, ClusterInfo, + DecommissionPoolStatus, DecommissionStatus, DiskInfo, HealDriveInfo, HealDriveInfos, + HealResultItem, HealRuntimeState, HealScanMode, HealStartRequest, HealStatus, HealTaskRequest, + HealingDiskInfo, MemStats, ObjectsInfo, PoolDecommissionInfo, PoolErasureSetInfo, PoolStatus, + PoolTarget, RebalanceCleanupWarnings, RebalancePoolProgress, RebalancePoolStatus, + RebalanceStartResult, RebalanceStatus, ServerInfo, UsageInfo, }; pub use configuration::{ ConfigApi, ConfigChange, ConfigDiff, ConfigDocument, ConfigHelp, ConfigHelpEntry, diff --git a/crates/s3/src/admin.rs b/crates/s3/src/admin.rs index c3319281..245c7c80 100644 --- a/crates/s3/src/admin.rs +++ b/crates/s3/src/admin.rs @@ -16,14 +16,14 @@ use bytes::Bytes; use futures::StreamExt; use rc_core::admin::{ AccessKeyInfo, AccessKeyKind, AccessKeyProvider, AccessKeyRecord, AccountApi, AccountInfo, - AccountMfaApi, AdminApi, BucketMetadataApi, BucketMetadataArchive, BucketQuota, - BulkAccessKeyApi, BulkAccessKeyQuery, CapabilityApi, CapabilityAvailability, CapabilityEntry, - CapabilityReport, ClusterInfo, ClusterSnapshotDocument, ClusterSnapshotMetadata, - ClusterSnapshotSummary, ConfigApi, ConfigDocument, ConfigHelp, ConfigHistoryEntry, - ConfigMutationResult, CreateServiceAccountRequest, DecommissionPoolStatus, DecommissionStatus, - DetailedHealthSnapshot, DiagnosticCapability, DiagnosticReadApi, EncryptedInspectArchive, - ExtensionsCatalog, Group, GroupStatus, HealRuntimeState, HealScanMode, HealStartRequest, - HealStatus, HealTaskRequest, IAM_ACCESS_KEYS_BULK_CAPABILITY, + AccountMfaApi, AdminApi, BackgroundHealCoverage, BucketMetadataApi, BucketMetadataArchive, + BucketQuota, BulkAccessKeyApi, BulkAccessKeyQuery, CapabilityApi, CapabilityAvailability, + CapabilityEntry, CapabilityReport, ClusterInfo, ClusterSnapshotDocument, + ClusterSnapshotMetadata, ClusterSnapshotSummary, ConfigApi, ConfigDocument, ConfigHelp, + ConfigHistoryEntry, ConfigMutationResult, CreateServiceAccountRequest, DecommissionPoolStatus, + DecommissionStatus, DetailedHealthSnapshot, DiagnosticCapability, DiagnosticReadApi, + EncryptedInspectArchive, ExtensionsCatalog, Group, GroupStatus, HealRuntimeState, HealScanMode, + HealStartRequest, HealStatus, HealTaskRequest, IAM_ACCESS_KEYS_BULK_CAPABILITY, IAM_ACCESS_KEYS_BULK_LDAP_CAPABILITY, IAM_ACCESS_KEYS_BULK_OPENID_CAPABILITY, IAM_POLICY_DETACH_CAPABILITY, IAM_POLICY_ENTITIES_CAPABILITY, INSPECT_ARCHIVE_COMPLETION, INSPECT_ARCHIVE_CONTENT_TYPE, IamArchiveApi, IamArchiveImportResult, IamArchiveImportSection, @@ -1964,6 +1964,10 @@ struct BackgroundHealStatusResponse { #[serde(default)] state: Option, #[serde(default)] + cluster_status_complete: Option, + #[serde(default)] + coverage: Option, + #[serde(default)] bitrot_start_time: Option, #[serde(default)] bitrot_start_cycle: u64, @@ -2043,11 +2047,17 @@ impl From for HealStatus { | HealRuntimeState::Uninitialized | HealRuntimeState::Idle, ) => false, - Some(HealRuntimeState::Unknown) | None => legacy_status_healing, + // Unknown runtime state does not make legacy scan timestamps authoritative. + Some(HealRuntimeState::Degraded | HealRuntimeState::Unknown) => { + queue_length > 0 || active_tasks > 0 + } + None => legacy_status_healing, }; let mut status = Self { healing, + cluster_status_complete: response.cluster_status_complete, + coverage: response.coverage, started: response.bitrot_start_time, scan_mode, scan_cycle: response.bitrot_start_cycle, @@ -7854,6 +7864,8 @@ mod tests { fn test_background_heal_status_response_maps_to_heal_status() { let status = HealStatus::from(BackgroundHealStatusResponse { state: Some(HealRuntimeState::Active), + cluster_status_complete: None, + coverage: None, bitrot_start_time: Some("2026-04-19T10:00:00Z".to_string()), bitrot_start_cycle: 42, current_scan_mode: Some(2), @@ -7873,6 +7885,8 @@ mod tests { let idle = HealStatus::from(BackgroundHealStatusResponse { state: Some(HealRuntimeState::Idle), + cluster_status_complete: None, + coverage: None, bitrot_start_time: None, bitrot_start_cycle: 0, current_scan_mode: Some(1), @@ -7888,6 +7902,8 @@ mod tests { let active_without_legacy_counters = HealStatus::from(BackgroundHealStatusResponse { state: Some(HealRuntimeState::Active), + cluster_status_complete: None, + coverage: None, bitrot_start_time: None, bitrot_start_cycle: 0, current_scan_mode: Some(1), @@ -7900,6 +7916,8 @@ mod tests { let disabled_with_stale_counters = HealStatus::from(BackgroundHealStatusResponse { state: Some(HealRuntimeState::Disabled), + cluster_status_complete: None, + coverage: None, bitrot_start_time: Some("2026-04-19T10:00:00Z".to_string()), bitrot_start_cycle: 42, current_scan_mode: Some(2), @@ -7912,6 +7930,8 @@ mod tests { let completed = HealStatus::from(BackgroundHealStatusResponse { state: None, + cluster_status_complete: None, + coverage: None, bitrot_start_time: Some("2026-04-19T10:00:00Z".to_string()), bitrot_start_cycle: 42, current_scan_mode: Some(1), @@ -7926,6 +7946,8 @@ mod tests { let legacy = HealStatus::from(BackgroundHealStatusResponse { state: None, + cluster_status_complete: None, + coverage: None, bitrot_start_time: Some("2026-04-19T10:00:00Z".to_string()), bitrot_start_cycle: 0, current_scan_mode: None, @@ -7938,6 +7960,8 @@ mod tests { let active = HealStatus::from(BackgroundHealStatusResponse { state: None, + cluster_status_complete: None, + coverage: None, bitrot_start_time: None, bitrot_start_cycle: 0, current_scan_mode: None, diff --git a/schemas/output_v2.json b/schemas/output_v2.json index 6fa40bc6..226fdab4 100644 --- a/schemas/output_v2.json +++ b/schemas/output_v2.json @@ -260,7 +260,7 @@ }, "healing": { "type": "boolean", - "description": "Whether a heal operation is active" + "description": "Whether heal work is known to be active. False does not prove cluster idleness when status is unknown or incomplete." }, "state": { "type": "string", @@ -269,10 +269,40 @@ "uninitialized", "idle", "active", + "degraded", "unknown" ], "description": "Runtime state reported by the heal service" }, + "clusterStatusComplete": { + "type": "boolean", + "description": "Whether the server has complete cluster status. Absent on legacy servers and token-scoped responses." + }, + "coverage": { + "type": "object", + "description": "Node coverage of the background heal snapshot. Missing counts mean unknown, not zero.", + "properties": { + "expected": { + "type": "integer", + "minimum": 0 + }, + "responded": { + "type": "integer", + "minimum": 0 + }, + "unknown": { + "type": "integer", + "minimum": 0 + }, + "reasons": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Coverage reason codes, including codes from future servers." + } + } + }, "summary": { "type": "string", "description": "Token-scoped manual heal task summary"