From ef1958caddd11a0a6f29ab3255b08bf5c7194b66 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Mon, 26 Oct 2026 08:57:56 +0000 Subject: [PATCH] fix(privacy): protect YAML keys and BOM-prefixed documents Restack unique #4527 onto the rewritten #4485 tip. --- .../tracedecay-privacy/src/structural_id.rs | 36 ++- crates/tracedecay-privacy/src/structured.rs | 3 +- .../src/structured_tests.rs | 47 +++- .../tracedecay-privacy/src/structured_text.rs | 47 +++- .../src/structured_text_tests.rs | 224 ++++++++++++++++++ 5 files changed, 342 insertions(+), 15 deletions(-) diff --git a/crates/tracedecay-privacy/src/structural_id.rs b/crates/tracedecay-privacy/src/structural_id.rs index b11a740a86..5d9397c4ad 100644 --- a/crates/tracedecay-privacy/src/structural_id.rs +++ b/crates/tracedecay-privacy/src/structural_id.rs @@ -1,7 +1,7 @@ use tracedecay_domain::canonical_text::is_tagged_lowercase_hex; use super::length_prefixed_sha256_hex; -use super::structured_text::sanitize_provider_metadata_text; +use super::structured_text::sanitize_structured_text; const PROTECTION_DOMAIN_V1: &[u8] = b"tracedecay.structural-identity-protection.v1"; const PROTECTION_PREFIX_V1: &str = "privacy.structural-id.v1."; @@ -28,8 +28,13 @@ pub fn protect_sensitive_structural_id(value: &str) -> Result sanitize_parsed(value, limits), Err(JsonPreflightFailureV1::Malformed) => sanitize_malformed(text, limits), // The preflight refuses before `Value` materializes, so it is the only diff --git a/crates/tracedecay-privacy/src/structured_tests.rs b/crates/tracedecay-privacy/src/structured_tests.rs index c6dff8a23a..c8589712b8 100644 --- a/crates/tracedecay-privacy/src/structured_tests.rs +++ b/crates/tracedecay-privacy/src/structured_tests.rs @@ -1,5 +1,6 @@ use super::structured::{ - StructuredSanitizationError, StructuredSanitizationLimits, sanitize_structured_payload, + StructuredSanitizationError, StructuredSanitizationLimits, sanitize_provider_metadata_json, + sanitize_structured_payload, }; const SECRET: &str = "sk-test-123456789012345678901234"; @@ -91,6 +92,50 @@ fn structured_limits_deny_raw_expansion_depth_and_item_overruns() { ); } +#[test] +fn bom_json_parsing_preserves_raw_byte_limits_and_metadata_redaction() { + let raw = r#"{"safe":"kept"}"#; + let with_bom = format!("\u{feff}{raw}"); + let plain_limits = + StructuredSanitizationLimits::new(raw.len(), 128, 8, 8).expect("valid limits"); + let plain = sanitize_structured_payload(raw.as_bytes(), plain_limits) + .expect("unprefixed input fits its exact byte allowance"); + assert!(plain.was_structurally_parsed()); + assert_eq!( + sanitize_structured_payload(with_bom.as_bytes(), plain_limits).unwrap_err(), + StructuredSanitizationError::RawBytesExceeded, + "the BOM's three original bytes still count toward raw admission" + ); + + let bom_limits = + StructuredSanitizationLimits::new(with_bom.len(), 128, 8, 8).expect("valid limits"); + let prefixed = sanitize_structured_payload(with_bom.as_bytes(), bom_limits) + .expect("BOM input fits its exact byte allowance"); + assert!(prefixed.was_structurally_parsed()); + assert_eq!(prefixed.payload(), plain.payload()); + assert_eq!( + sanitize_provider_metadata_json(&with_bom, raw.len() as u64), + None + ); + assert_eq!( + sanitize_provider_metadata_json(&with_bom, with_bom.len() as u64).as_ref(), + Some(plain.payload()) + ); + + let private_metadata = "\u{feff}{\"vault_passphrase\":\"ordinary-value\",\"safe\":\"kept\"}"; + assert_eq!( + sanitize_provider_metadata_json(private_metadata, 4096), + Some(serde_json::json!({ + "vault_passphrase": "[TraceDecay redacted: sensitive field]", + "safe": "kept" + })) + ); + assert_eq!( + sanitize_structured_payload(b"\xef\xbb\xbf{\"key\":1,\"key\":2}", limits()).unwrap_err(), + StructuredSanitizationError::UnsafeJsonStructure + ); +} + /// Pins the format a table-headed document is classified as, separately from /// what the sanitizer then does with it. The sanitizer reports "no format" both /// when nothing claimed the document and when it was quarantined, so a failure diff --git a/crates/tracedecay-privacy/src/structured_text.rs b/crates/tracedecay-privacy/src/structured_text.rs index 60cb97d8e8..109fd1fa31 100644 --- a/crates/tracedecay-privacy/src/structured_text.rs +++ b/crates/tracedecay-privacy/src/structured_text.rs @@ -41,12 +41,15 @@ use super::detector_kernel::{NormalizedSensitiveKey, SensitiveKeyPolicy}; use super::length_prefixed_sha256_hex; use super::rules::{CredentialPatternSet, CredentialScanBatchV1}; use super::structured::{ - ParsedStructuredTextV1, StructuredSanitizationError, StructuredSanitizationLimits, - StructuredTextFieldV1, StructuredTextFormatV1, StructuredTextParseFailureV1, - parse_structured_text, sanitize_structured_payload, validate_structured_text_limits, + MAX_STRUCTURED_TEXT_BYTES, ParsedStructuredTextV1, StructuredSanitizationError, + StructuredSanitizationLimits, StructuredTextFieldV1, StructuredTextFormatV1, + StructuredTextParseFailureV1, parse_structured_text, sanitize_structured_payload, + validate_structured_text_limits, }; -pub const CODE_SOURCE_SANITIZER_VERSION_V1: &str = "privacy.code-source.v1"; +// Retained generations must not reuse content admitted before key and BOM +// inspection. The receipt shape and identity domain remain V1. +pub const CODE_SOURCE_SANITIZER_VERSION_V1: &str = "privacy.code-source.v1.structured-keys.1"; const CODE_SOURCE_RECEIPT_DOMAIN_V1: &[u8] = b"tracedecay.privacy.code-source.receipt.v1\0"; pub const LCM_PAYLOAD_SANITIZER_VERSION_V1: &str = "privacy.lcm-payload.v1"; const LCM_PAYLOAD_RECEIPT_DOMAIN_V1: &[u8] = b"tracedecay.privacy.lcm-payload.receipt.v1\0"; @@ -171,7 +174,16 @@ pub(crate) fn sanitize_structured_text( raw: &str, ) -> Result { let patterns = credential_patterns()?; - let sanitized = sanitize_structured_text_with(raw, patterns)?; + if raw.len() > MAX_STRUCTURED_TEXT_BYTES { + return Err(DetectionError::ScanLimitExceeded); + } + // Parsing, span discovery, and rewrite verification must use the same + // coordinates. Restore the BOM only after those operations finish. + let document = raw.strip_prefix('\u{feff}').unwrap_or(raw); + let mut sanitized = sanitize_structured_text_with(document, patterns)?; + if document.len() != raw.len() { + sanitized.sanitized_text.insert(0, '\u{feff}'); + } patterns .checked(sanitized) .map_err(|_| DetectionError::Initialization) @@ -202,7 +214,13 @@ fn sanitize_structured_text_with( tree_candidates(raw, &parsed, &policy, patterns, &mut quarantine_findings) } else { ( - line_candidates(raw, &parsed.fields, &policy, patterns), + line_candidates( + raw, + &parsed.fields, + &policy, + patterns, + &mut quarantine_findings, + ), RedactedKeys::default(), ) }; @@ -406,9 +424,18 @@ fn line_candidates( fields: &[StructuredTextFieldV1], policy: &ConfiguredSensitiveKeyPolicy<'_>, patterns: &CredentialPatternSet, + quarantine_findings: &mut Vec, ) -> Vec { let mut candidates = Vec::new(); for field in fields { + let mut key_evidence = field.key.clone(); + redact_text( + &mut key_evidence, + "$", + patterns, + quarantine_findings, + SanitizationActionV1::Quarantined, + ); if field.value_span.start >= field.value_span.end || field.value_span.end > raw.len() || !raw.is_char_boundary(field.value_span.start) @@ -1174,9 +1201,9 @@ pub fn sanitize_code_source_bytes( /// detector change must invalidate a completed-rescan watermark. pub fn lcm_payload_detector_revision() -> &'static str { // Advance this input when detector semantics change independently of the - // rule documents. Restored strict entropy must recheck previously exempt - // compound tokens without changing the receipt contract. - const ALGORITHM_REVISION: &[u8] = b"strict-entropy.1"; + // rule documents. Key and BOM inspection must recheck payloads admitted + // by the prior implementation without changing the receipt contract. + const ALGORITHM_REVISION: &[u8] = b"strict-entropy.1+structured-keys.1"; static REVISION: OnceLock = OnceLock::new(); REVISION.get_or_init(|| { let [catalogue, supplement] = super::rules::rule_document_bytes(); @@ -1262,7 +1289,7 @@ fn detect_lcm_payload(raw: &str) -> Result<(String, Vec), if raw.len() > MAX_LCM_PAYLOAD_BYTES_V1 { return Err(DetectionError::ScanLimitExceeded); } - let trimmed = raw.trim_start(); + let trimmed = raw.strip_prefix('\u{feff}').unwrap_or(raw).trim_start(); let json_container = trimmed.starts_with('{') || trimmed .strip_prefix('[') diff --git a/crates/tracedecay-privacy/src/structured_text_tests.rs b/crates/tracedecay-privacy/src/structured_text_tests.rs index cbb28610fc..1b30d64234 100644 --- a/crates/tracedecay-privacy/src/structured_text_tests.rs +++ b/crates/tracedecay-privacy/src/structured_text_tests.rs @@ -506,6 +506,230 @@ fn lcm_json_credential_bearing_keys_quarantine_instead_of_faulting_the_receipt() ); } +fn assert_public_credential_key_protection(raw: &str, credential_key: &str) { + assert_eq!(raw_sweep(PLACEHOLDER), PLACEHOLDER); + assert!(raw_sweep(raw).contains(PLACEHOLDER)); + assert!(!raw_sweep(credential_key).contains(credential_key)); + assert!( + sanitize_structured_text(raw) + .expect("valid structured document is scanned") + .format() + .is_some(), + "a leading BOM must not bypass structural parsing" + ); + + let assert_safe_output = |text: &str| { + assert!(!text.contains(credential_key), "credential key was exposed"); + assert!(!text.contains(PLACEHOLDER), "the key's value was exposed"); + assert!(text.contains("us-east"), "unrelated field was discarded"); + }; + let assert_safe_evidence = |evidence: &str| { + assert!(!evidence.contains(credential_key)); + assert!(!evidence.contains(PLACEHOLDER)); + }; + let assert_key_refusal = |error: DetectionError| { + assert_eq!(error, DetectionError::CredentialKeyQuarantine); + assert_safe_evidence(&error.to_string()); + }; + + match sanitize_code_source_bytes(raw.as_bytes(), CodeSourceShapeV1::StructuredData) { + Ok(sanitized) => { + let (bytes, receipt) = sanitized.into_parts(); + let text = String::from_utf8(bytes).expect("sanitized UTF-8"); + assert_safe_output(&text); + assert_safe_evidence(&serde_json::to_string(&receipt).expect("source receipt")); + let replayed = + sanitize_code_source_bytes(text.as_bytes(), CodeSourceShapeV1::StructuredData) + .expect("redacted source remains usable"); + assert_eq!(replayed.sanitized_bytes(), text.as_bytes()); + } + Err(error) => assert_key_refusal(error), + } + + match sanitize_lcm_payload_text(raw) { + Ok(sanitized) => { + let (text, receipt, findings) = sanitized.into_parts(); + assert_safe_output(&text); + assert_safe_evidence( + &serde_json::to_string(&(receipt, findings)).expect("LCM evidence"), + ); + let replayed = sanitize_lcm_payload_text(&text).expect("redacted LCM stays usable"); + assert_eq!(replayed.sanitized_text(), text); + } + Err(error) => assert_key_refusal(error), + } + + if let Some(text) = sanitize_provider_metadata_text(raw) { + assert_safe_output(&text); + assert_eq!( + sanitize_provider_metadata_text(&text).as_deref(), + Some(text.as_str()), + "redacted metadata must be a fixed point" + ); + } +} + +#[test] +fn yaml_credential_keys_protect_plain_descendants_through_public_sanitizers() { + let credential_key = ["sk", "-test-", "1234567890abcdef"].concat(); + const ALPHABET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + let entropy_key: String = (0..48) + .map(|index| char::from(ALPHABET[(index * 17) % ALPHABET.len()])) + .collect(); + for (key, raw) in [ + // This also has valid HTTP-header syntax; that route must protect keys. + ( + credential_key.as_str(), + format!("{credential_key}: {PLACEHOLDER}\nregion: us-east\n"), + ), + ( + entropy_key.as_str(), + format!("{entropy_key}: {PLACEHOLDER}\nregion: us-east\n"), + ), + ( + credential_key.as_str(), + format!( + "provider:\n {credential_key}:\n accounts:\n - {PLACEHOLDER}\nregion: us-east\n" + ), + ), + ] { + assert_public_credential_key_protection(&raw, key); + } +} + +#[test] +fn empty_credential_key_fields_still_require_key_quarantine() { + let credential_key = ["sk", "-test-", "1234567890abcdef"].concat(); + let raw = format!("{credential_key}:\nregion: us-east\n"); + + assert_eq!( + sanitize_code_source_bytes(raw.as_bytes(), CodeSourceShapeV1::StructuredData) + .map(|_| ()) + .expect_err("empty value does not make its credential key safe"), + DetectionError::CredentialKeyQuarantine + ); + assert_eq!( + sanitize_lcm_payload_text(&raw).expect_err("LCM withholds credential key"), + DetectionError::CredentialKeyQuarantine + ); + assert!(sanitize_provider_metadata_text(&raw).is_none()); +} + +#[test] +fn bom_credential_keys_protect_plain_descendants_in_json_and_toml() { + let credential_key = ["sk", "-test-", "1234567890abcdef"].concat(); + for raw in [ + format!( + "\u{feff}{{\"{credential_key}\":{{\"accounts\":[\"{PLACEHOLDER}\"]}},\"region\":\"us-east\"}}" + ), + format!( + "\u{feff}region = \"us-east\"\n[{credential_key}]\naccounts = [\"{PLACEHOLDER}\"]\n" + ), + ] { + assert_public_credential_key_protection(&raw, &credential_key); + } +} + +#[test] +fn bom_sensitive_fields_keep_exact_source_offsets_and_neighbors() { + for (format, raw, expected) in [ + ( + StructuredTextFormatV1::Json, + format!( + "\u{feff}{{\"note\":\"café\", \"vault_passphrase\":\"{PLACEHOLDER}\", \"region\":\"us-east\"}}\n" + ), + "\u{feff}{\"note\":\"café\", \"vault_passphrase\":\"TraceDecay-redacted-sensitive-field\", \"region\":\"us-east\"}\n", + ), + ( + StructuredTextFormatV1::Toml, + format!( + "\u{feff}# café\n[vault]\nregion = \"us-east\"\nvault_passphrase = \"{PLACEHOLDER}\" # rotate\n" + ), + "\u{feff}# café\n[vault]\nregion = \"us-east\"\nvault_passphrase = \"TraceDecay-redacted-sensitive-field\" # rotate\n", + ), + ] { + assert_detected_only_by_parsing(&raw, format, expected); + let source = sanitize_code_source_bytes(raw.as_bytes(), CodeSourceShapeV1::StructuredData) + .expect("BOM source sanitizes"); + assert_eq!(source.sanitized_bytes(), expected.as_bytes()); + assert_eq!( + sanitize_provider_metadata_text(&raw).as_deref(), + Some(expected) + ); + let lcm = sanitize_lcm_payload_text(&raw).expect("BOM payload sanitizes"); + assert!(!lcm.sanitized_text().contains(PLACEHOLDER)); + assert!(lcm.sanitized_text().contains("us-east")); + } +} + +#[test] +fn bom_ordinary_structured_documents_preserve_nonsecret_content() { + for raw in [ + "\u{feff}{\"note\":\"café\", \"region\":\"us-east\"}\n", + "\u{feff}# café\n[service]\nregion = \"us-east\"\n", + "region: us-east\ncomponent: worker\n", + ] { + let source = sanitize_code_source_bytes(raw.as_bytes(), CodeSourceShapeV1::StructuredData) + .expect("ordinary source sanitizes"); + assert_eq!(source.sanitized_bytes(), raw.as_bytes()); + assert_eq!(sanitize_provider_metadata_text(raw).as_deref(), Some(raw)); + let lcm = sanitize_lcm_payload_text(raw).expect("ordinary LCM payload sanitizes"); + assert!(lcm.sanitized_text().contains("us-east")); + assert!(lcm.findings().is_empty()); + } +} + +#[test] +fn bom_malformed_structured_documents_cannot_fall_back_to_raw_scanning() { + for raw in [ + format!("\u{feff}{{\"vault_passphrase\":\"{PLACEHOLDER}\",\"broken\":["), + format!("\u{feff}[vault]\nvault_passphrase = \"{PLACEHOLDER}\"\nbroken = [\n"), + format!( + "\u{feff}{{\"vault_passphrase\":\"{PLACEHOLDER}\",\"vault_passphrase\":\"safe-replacement\"}}" + ), + ] { + assert!(raw_sweep(&raw).contains(PLACEHOLDER)); + assert_eq!( + sanitize_code_source_bytes(raw.as_bytes(), CodeSourceShapeV1::StructuredData) + .map(|_| ()) + .expect_err("malformed BOM source is withheld"), + DetectionError::StructuredQuarantine + ); + assert_eq!( + sanitize_lcm_payload_text(&raw).expect_err("malformed BOM payload is withheld"), + DetectionError::StructuredQuarantine + ); + assert!(sanitize_provider_metadata_text(&raw).is_none()); + } +} + +#[test] +fn bom_lcm_json_keeps_its_existing_budget_and_original_input_receipt() { + // JSON payloads already have a larger LCM allowance than general structured + // text. A BOM must not send valid JSON through the smaller text route. + let raw = format!( + "{{\"text\":\"{}\",\"region\":\"us-east\"}}", + "a".repeat(1_048_576) + ); + let with_bom = format!("\u{feff}{raw}"); + let (plain_text, plain_receipt, plain_findings) = sanitize_lcm_payload_text(&raw) + .expect("JSON fits the existing LCM payload allowance") + .into_parts(); + let (bom_text, bom_receipt, bom_findings) = sanitize_lcm_payload_text(&with_bom) + .expect("BOM does not change the JSON allowance") + .into_parts(); + + assert_eq!(bom_text, plain_text); + assert_eq!(bom_findings, plain_findings); + assert!(bom_findings.is_empty()); + assert_eq!(bom_receipt.payload(), plain_receipt.payload()); + assert_ne!( + bom_receipt.receipt(), + plain_receipt.receipt(), + "receipt identity must retain the original input's BOM bytes" + ); +} + #[test] fn prose_opening_with_a_bracketed_annotation_line_is_not_toml_quarantine() { // LCM compaction emits scaffold messages such as