diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fb0233b03..4acdc9270 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -214,9 +214,14 @@ jobs: run: | cargo clippy --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" --all-targets -- -D warnings + - name: Set up Chrome for browser fixture tests + id: setup-chrome + uses: browser-actions/setup-chrome@v1 + - name: cargo test - run: | - cargo test --manifest-path crates/trusted-server-cli/Cargo.toml --target "$(rustc -vV | sed -n 's/host: //p')" + run: ./scripts/test-cli.sh + env: + CHROME: ${{ steps.setup-chrome.outputs.chrome-path }} test-typescript: name: vitest diff --git a/Cargo.lock b/Cargo.lock index cb8f40c68..cb467cefa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5322,8 +5322,11 @@ dependencies = [ "derive_more", "directories", "edgezero-cli", + "edgezero-core", "error-stack", "futures", + "glob", + "http", "http-body-util", "hyper", "hyper-util", @@ -5335,12 +5338,15 @@ dependencies = [ "scraper", "serde", "serde_json", + "similar", + "temp-env", "tempfile", "time", "tokio", "tokio-rustls", "toml", "toml_edit", + "tracing", "trusted-server-core", "url", "webpki-roots", diff --git a/Cargo.toml b/Cargo.toml index 7ca87e687..a7fd306c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,6 +94,7 @@ scraper = "0.24.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.149" sha2 = "0.10.9" +similar = "2.7" simple_logger = "5" spin-sdk = { version = "~6.0", default-features = false, features = ["http", "key-value", "variables"] } subtle = "2.6" @@ -106,6 +107,7 @@ tokio-rustls = "0.26" toml = "1.1" toml_edit = "0.23.10" tower = "0.4" +tracing = "0.1" trusted-server-core = { path = "crates/trusted-server-core" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } diff --git a/README.md b/README.md index b87fe61ad..81794720c 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ ts config init ts config validate # Audit a public page with Chrome/Chromium to bootstrap a draft config -ts audit https://publisher.example +ts audit generate https://publisher.example # Run tests (Fastly/WASM crates — requires Viceroy) cargo test-fastly diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index fe9c3664b..e08114850 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -18,16 +18,21 @@ workspace = true chromiumoxide = { workspace = true } clap = { workspace = true } edgezero-cli = { workspace = true } +edgezero-core = { workspace = true } futures = { workspace = true } +glob = { workspace = true } +http = { workspace = true } log = { workspace = true } regex = { workspace = true } scraper = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +similar = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true } toml = { workspace = true } toml_edit = { workspace = true } +tracing = { workspace = true } trusted-server-core = { workspace = true } url = { workspace = true } which = { workspace = true } @@ -62,4 +67,5 @@ tokio = { workspace = true, features = ["test-util"] } x509-parser = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +temp-env = { workspace = true } tempfile = { workspace = true } diff --git a/crates/trusted-server-cli/src/ad_templates/compare.rs b/crates/trusted-server-cli/src/ad_templates/compare.rs new file mode 100644 index 000000000..48ab71f3e --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/compare.rs @@ -0,0 +1,830 @@ +//! Pure comparison of configured expected slots against browser ad evidence. +//! +//! This module is collector-independent and Chrome-free: it takes decoded +//! [`BrowserAdEvidence`] plus the [`ExpectedSlot`] set and produces a +//! [`PageVerificationResult`] with per-slot statuses, warnings, and unmatched +//! extra evidence, mirroring spec §5.3–§5.6. +//! +use serde::Deserialize; + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + +use crate::ad_templates::expected::ExpectedSlot; +use crate::ad_templates::output::Warning; + +/// The phase in which a piece of evidence was observed. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidencePhase { + /// Observed during the initial load and settle. + InitialLoad, + /// Observed only after the deterministic scroll pass. + Scroll, +} + +/// A DOM element ID observed on the page. +#[derive(Debug, Clone, Deserialize)] +pub struct DomEvidence { + /// The element ID. + pub dom_id: String, + /// The phase it was first observed in. + pub phase: EvidencePhase, +} + +/// A GPT slot observed on the page. +#[derive(Debug, Clone, Deserialize)] +pub struct GptSlotEvidence { + /// The observed GAM ad unit path. + pub gam_unit_path: String, + /// The observed GPT slot element ID. + pub div_id: String, + /// Observed numeric sizes as `(width, height)` pairs (non-numeric dropped upstream). + pub sizes: Vec<(u32, u32)>, + /// The phase it was first observed in. + pub phase: EvidencePhase, +} + +/// An `apstag.fetchBids` call the page made, if any were recorded. +/// +/// The collector no longer hooks `apstag`: server-side APS configuration is +/// metadata rather than a client assertion, so a missing client call is not a +/// finding. The field and this shape stay for the evidence payload's schema, and +/// the list arrives empty. +#[derive(Debug, Clone, Deserialize)] +#[allow( + dead_code, + reason = "decoded for schema stability; the collector records no APS calls" +)] +pub struct ApsFetchBidsEvidence { + /// The APS slot ID requested. + pub slot_id: String, + /// Sizes requested for the slot. + pub sizes: Vec<(u32, u32)>, + /// The phase it was observed in. + pub phase: EvidencePhase, +} + +/// A `/__ts/page-bids` observation for SPA routes (spec §5.2). +/// +/// DEFERRED in Phase 1: kept as forward scaffolding so the decoded evidence shape +/// stays forward-compatible. Not populated by the collector or surfaced in JSON. +#[derive(Debug, Clone, Deserialize)] +#[allow( + dead_code, + reason = "reserved decoded shape for the optional bids phase" +)] +pub struct PageBidsEvidence { + /// The slot ID present in the page-bids response. + pub slot_id: String, + /// The phase it was observed in. + pub phase: EvidencePhase, +} + +/// All read-only ad evidence decoded from a single browser page. +#[derive(Debug, Clone, Deserialize)] +pub struct BrowserAdEvidence { + /// DOM element IDs matching configured prefixes. + pub dom_ids: Vec, + /// GPT slots observed via `defineSlot` and `getSlots()`. + pub gpt_slots: Vec, + /// `apstag.fetchBids` calls observed. + pub aps_calls: Vec, + /// `/__ts/page-bids` observations (deferred; default empty). + #[serde(default)] + #[allow(dead_code, reason = "reserved for the optional bids phase")] + pub page_bids: Vec, + /// Collector-level warnings (no page HTML/cookies/storage). + #[serde(default)] + pub warnings: Vec, +} + +/// Summary of the runtime ad-stack gate for a page. +#[derive(Debug, Clone, Copy)] +pub struct RuntimeGateSummary { + /// The three-state ad-stack expectation. + pub expected: RuntimeAdStackExpected, +} + +impl RuntimeGateSummary { + /// Builds a summary from a computed runtime expectation. + #[must_use] + pub fn from_expected(expected: RuntimeAdStackExpected) -> Self { + Self { expected } + } + + #[cfg(test)] + fn unknown_allowed() -> Self { + Self::from_expected(RuntimeAdStackExpected::Unknown) + } + + #[cfg(test)] + fn auction_disabled() -> Self { + Self::from_expected(RuntimeAdStackExpected::No) + } +} + +/// Confirmation status for a single configured slot (compare-side mirror of the +/// output `SlotStatus`). +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum SlotStatus { + /// GPT evidence matches GAM path, div, and a compatible size. + Confirmed, + /// Some evidence, but not enough to confirm. + Partial, + /// No DOM or GPT evidence confirms the slot. + Missing, + /// The checker cannot confirm this slot type; this is not page drift. + Unconfirmable, +} + +/// The verification result for one audited page. +#[derive(Debug, Clone)] +pub struct PageVerificationResult { + /// Whether the runtime ad stack was expected to run for this page. + pub runtime_ad_stack_expected: RuntimeAdStackExpected, + /// Per-slot results, in expected-slot order. + pub slots: Vec, + /// Live evidence that matched no configured slot. + pub extra_evidence: Vec, +} + +impl PageVerificationResult { + /// Whether `--strict` should fail for this page. + /// + /// False when the runtime ad stack is not expected to run (a known gate + /// suppressed it); otherwise true if any slot is missing or partial. Provider + /// warnings and extra evidence alone never fail strict. + #[must_use] + pub fn strict_failed(&self) -> bool { + if self.runtime_ad_stack_expected == RuntimeAdStackExpected::No { + return false; + } + self.slots + .iter() + .any(|slot| matches!(slot.status, SlotStatus::Missing | SlotStatus::Partial)) + } +} + +/// Per-slot verification result. +#[derive(Debug, Clone)] +pub struct SlotResult { + /// The configured slot id. + pub id: String, + /// The confirmation status. + pub status: SlotStatus, + /// The phase the confirming evidence was observed in. + pub phase: Option, + /// The live evidence observed for this slot. + pub evidence: SlotEvidence, + /// Slot-level warnings (size, provider, etc.). + pub warnings: Vec, +} + +/// Live evidence observed for a configured slot. +#[derive(Debug, Clone)] +pub struct SlotEvidence { + /// The resolved DOM element ID, if any. + pub dom_id: Option, + /// The matched GPT slot, if any. + pub gpt: Option, +} + +/// Live ad-slot evidence with no matching configured slot. +#[derive(Debug, Clone)] +pub struct ExtraEvidence { + /// Evidence kind. Only `gpt` is produced today; the field is a string so a + /// later evidence source can be added without changing the JSON schema. + pub kind: String, + /// The phase it was observed in. + pub phase: EvidencePhase, + /// The DOM element ID, if any. + pub dom_id: Option, + /// The GAM unit path, if any. + pub gam_unit_path: Option, + /// Observed numeric sizes. + pub sizes: Vec<(u32, u32)>, + /// Why this evidence is reported as extra. + pub reason: String, +} + +fn warning(code: &str, message: String) -> Warning { + Warning { + code: code.to_string(), + message, + } +} + +/// Resolves the slot root DOM element per spec §5.3. +/// +/// Exact `div_id` match first, then the first element whose ID starts with +/// `div_id`, ignoring `-container` wrappers. +fn resolve_dom<'a>(dom_ids: &'a [DomEvidence], div_id: &str) -> Option<&'a DomEvidence> { + if let Some(exact) = dom_ids.iter().find(|dom| dom.dom_id == div_id) { + return Some(exact); + } + dom_ids + .iter() + .find(|dom| dom.dom_id.starts_with(div_id) && !dom.dom_id.ends_with("-container")) +} + +/// Returns true when a GPT slot's element ID matches the resolved DOM id (or its +/// `-container`), per spec §5.4. +fn gpt_div_matches(gpt_div: &str, expected: &ExpectedSlot, resolved_dom_id: Option<&str>) -> bool { + match resolved_dom_id { + Some(dom_id) => gpt_div == dom_id || gpt_div == format!("{dom_id}-container"), + None => { + gpt_div == expected.div_id + || (gpt_div.starts_with(&expected.div_id) && !gpt_div.ends_with("-container")) + } + } +} + +fn banner_sizes(expected: &ExpectedSlot) -> Vec<(u32, u32)> { + expected + .formats + .iter() + .filter(|format| format.media_type == MediaType::Banner) + .map(|format| (format.width, format.height)) + .collect() +} + +/// Compares configured expected slots against decoded browser evidence. +#[must_use] +pub fn compare_page_evidence( + expected: &[ExpectedSlot], + evidence: &BrowserAdEvidence, + gate: RuntimeGateSummary, +) -> PageVerificationResult { + let mut consumed_gpt = vec![false; evidence.gpt_slots.len()]; + let mut slots = Vec::with_capacity(expected.len()); + + for slot in expected { + let resolved = resolve_dom(&evidence.dom_ids, &slot.div_id); + let resolved_id = resolved.map(|dom| dom.dom_id.clone()); + // An unrenderable (`None`) configured path can never match live GPT + // evidence; matching on anything else would confirm the wrong unit. + let gpt_idx = slot.gam_unit_path.as_deref().and_then(|unit_path| { + evidence.gpt_slots.iter().position(|gpt| { + gpt.gam_unit_path == unit_path + && gpt_div_matches(&gpt.div_id, slot, resolved_id.as_deref()) + }) + }); + + let banner = banner_sizes(slot); + let mut warnings = Vec::new(); + // `expected_slots_for_path` drops a slot whose template does not render, + // so on the verify path this arm is unreachable; it exists for callers + // that build expected slots directly, and as a guard if that filter ever + // changes. + if slot.gam_unit_path.is_none() { + warnings.push(warning( + "gam_unit_path_unrenderable", + format!( + "slot `{}` gam_unit_path template renders past GAM's unit-path byte limit \ + for this page's section; the runtime omits this slot on this path", + slot.id + ), + )); + } + + let (status, dom_for_evidence, gpt_for_evidence, phase) = if let Some(idx) = gpt_idx { + consumed_gpt[idx] = true; + let gpt = &evidence.gpt_slots[idx]; + let dom_id = resolved_id.clone().or_else(|| Some(gpt.div_id.clone())); + if banner.is_empty() { + warnings.push(warning( + "unsupported_format", + format!( + "slot `{}` has only non-banner formats; not confirmable in Phase 1", + slot.id + ), + )); + ( + SlotStatus::Unconfirmable, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else if gpt.sizes.is_empty() { + warnings.push(warning( + "out_of_page_slot", + format!( + "slot `{}` matched an out-of-page GPT slot with no sizes", + slot.id + ), + )); + ( + SlotStatus::Partial, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else if banner.iter().any(|size| gpt.sizes.contains(size)) { + let extra: Vec<(u32, u32)> = gpt + .sizes + .iter() + .copied() + .filter(|size| !banner.contains(size)) + .collect(); + if !extra.is_empty() { + warnings.push(warning( + "extra_observed_size", + format!("slot `{}` observed extra GPT sizes {extra:?}", slot.id), + )); + } + let missing: Vec<(u32, u32)> = banner + .iter() + .copied() + .filter(|size| !gpt.sizes.contains(size)) + .collect(); + if !missing.is_empty() { + warnings.push(warning( + "configured_size_not_observed", + format!( + "slot `{}` configured sizes {missing:?} were not observed", + slot.id + ), + )); + } + ( + SlotStatus::Confirmed, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } else { + warnings.push(warning( + "incompatible_sizes", + format!( + "slot `{}` GPT path and div matched but no configured size overlapped", + slot.id + ), + )); + ( + SlotStatus::Partial, + dom_id, + Some(gpt.clone()), + Some(gpt.phase), + ) + } + } else if let Some(dom) = resolved { + warnings.push(warning( + "dom_without_gpt", + "DOM element matched, but no GPT slot evidence was observed".to_string(), + )); + ( + SlotStatus::Partial, + Some(dom.dom_id.clone()), + None, + Some(dom.phase), + ) + } else { + (SlotStatus::Missing, None, None, None) + }; + + slots.push(SlotResult { + id: slot.id.clone(), + status, + phase, + evidence: SlotEvidence { + dom_id: dom_for_evidence, + gpt: gpt_for_evidence, + }, + warnings, + }); + } + + let extra_evidence = evidence + .gpt_slots + .iter() + .enumerate() + .filter(|(idx, _)| !consumed_gpt[*idx]) + .map(|(_, gpt)| ExtraEvidence { + kind: "gpt".to_string(), + phase: gpt.phase, + dom_id: Some(gpt.div_id.clone()), + gam_unit_path: Some(gpt.gam_unit_path.clone()), + sizes: gpt.sizes.clone(), + reason: "no_configured_slot_matched".to_string(), + }) + .collect(); + + PageVerificationResult { + runtime_ad_stack_expected: gate.expected, + slots, + extra_evidence, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ad_templates::expected::ExpectedFormat; + + fn dom(id: &str) -> DomEvidence { + DomEvidence { + dom_id: id.to_string(), + phase: EvidencePhase::InitialLoad, + } + } + + fn gpt_slot(gam_unit_path: &str, div_id: &str, sizes: &[(u32, u32)]) -> GptSlotEvidence { + GptSlotEvidence { + gam_unit_path: gam_unit_path.to_string(), + div_id: div_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn aps(slot_id: &str, sizes: &[(u32, u32)]) -> ApsFetchBidsEvidence { + ApsFetchBidsEvidence { + slot_id: slot_id.to_string(), + sizes: sizes.to_vec(), + phase: EvidencePhase::InitialLoad, + } + } + + fn evidence( + doms: Vec, + gpts: Vec, + aps: Vec, + ) -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: doms, + gpt_slots: gpts, + aps_calls: aps, + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + fn expected_slot( + id: &str, + div_id: &str, + gam_unit_path: &str, + sizes: &[(u32, u32)], + providers: &[&str], + ) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), + formats: sizes + .iter() + .map(|&(width, height)| ExpectedFormat { + width, + height, + media_type: MediaType::Banner, + }) + .collect(), + providers: providers.iter().copied().map(String::from).collect(), + page_patterns: Vec::new(), + } + } + + fn expected_slot_video(id: &str, div_id: &str, gam_unit_path: &str) -> ExpectedSlot { + ExpectedSlot { + id: id.to_string(), + div_id: div_id.to_string(), + gam_unit_path: Some(gam_unit_path.to_string()), + formats: vec![ExpectedFormat { + width: 0, + height: 0, + media_type: MediaType::Video, + }], + providers: Vec::new(), + page_patterns: Vec::new(), + } + } + + #[test] + fn gpt_path_div_and_size_overlap_confirms_slot() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!( + result.slots[0].warnings.is_empty(), + "confirmed slot should carry no warnings" + ); + } + + #[test] + fn unrenderable_gam_unit_path_never_confirms() { + let mut expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + expected.gam_unit_path = None; + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Partial, + "an unrenderable configured path must not confirm against GPT evidence" + ); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "gam_unit_path_unrenderable"), + "should explain why the slot cannot be confirmed" + ); + } + + #[test] + fn dom_only_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(vec![dom("ad-atf-0")], Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "dom_without_gpt") + ); + } + + #[test] + fn no_dom_or_gpt_is_missing() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Missing); + } + + #[test] + fn prefix_dom_resolution_ignores_container_suffix() { + let expected = expected_slot( + "header", + "ad-header-0-", + "/123/homepage/header", + &[(728, 90)], + &[], + ); + let evidence = evidence( + vec![dom("ad-header-0--container"), dom("ad-header-0-_R_abc123")], + Vec::new(), + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].evidence.dom_id.as_deref(), + Some("ad-header-0-_R_abc123"), + "prefix match should skip -container" + ); + assert_eq!(result.slots[0].status, SlotStatus::Partial); + } + + #[test] + fn unmatched_gpt_slot_becomes_extra_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![ + gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)]), + gpt_slot( + "/123/publisher/right-rail", + "ad-right-rail-0", + &[(300, 250)], + ), + ], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert_eq!(result.extra_evidence.len(), 1); + assert_eq!(result.extra_evidence[0].kind, "gpt"); + assert!( + !result.strict_failed(), + "extra evidence alone must not fail strict" + ); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence(Vec::new(), Vec::new(), Vec::new()); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::auction_disabled(), + ); + + assert_eq!(result.runtime_ad_stack_expected, RuntimeAdStackExpected::No); + assert_eq!(result.slots[0].status, SlotStatus::Missing); + assert!( + !result.strict_failed(), + "missing slot must not fail strict when ad stack is No" + ); + } + + #[test] + fn gpt_incompatible_sizes_is_partial() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(728, 90)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "incompatible_sizes") + ); + } + + #[test] + fn non_banner_only_slot_is_unconfirmable_and_does_not_fail_strict() { + let expected = expected_slot_video("video", "ad-video-", "/123/news/video"); + let evidence = evidence( + vec![dom("ad-video-0")], + vec![gpt_slot("/123/news/video", "ad-video-0", &[(640, 480)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Unconfirmable); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "unsupported_format") + ); + assert!( + !result.strict_failed(), + "checker limitations should not fail strict" + ); + } + + #[test] + fn gpt_container_element_id_confirms() { + let expected = expected_slot("atf", "ad-atf-0", "/123/news/atf", &[(300, 250)], &[]); + let evidence = evidence( + vec![dom("ad-atf-0"), dom("ad-atf-0-container")], + vec![gpt_slot( + "/123/news/atf", + "ad-atf-0-container", + &[(300, 250)], + )], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Confirmed, + "container element id is a valid GPT div match" + ); + } + + #[test] + fn sizeless_live_slot_is_partial_when_config_declares_banner_sizes() { + let expected = expected_slot( + "interstitial", + "ad-oop-", + "/123/news/oop", + &[(300, 250)], + &[], + ); + let evidence = evidence( + vec![dom("ad-oop-0")], + vec![gpt_slot("/123/news/oop", "ad-oop-0", &[])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Partial); + assert!( + result.slots[0] + .warnings + .iter() + .any(|w| w.code == "out_of_page_slot") + ); + assert!( + result.strict_failed(), + "a live sizeless slot drifting from configured banner sizes must fail strict" + ); + } + + #[test] + fn aps_match_adds_no_warning() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + vec![aps("atf", &[(300, 250)])], + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!(result.slots[0].status, SlotStatus::Confirmed); + assert!( + !result.slots[0] + .warnings + .iter() + .any(|w| w.code.starts_with("aps_")), + "matching APS should not warn" + ); + } + + #[test] + fn server_side_aps_config_does_not_require_client_fetch_bids_evidence() { + let expected = expected_slot("atf", "ad-atf-", "/123/news/atf", &[(300, 250)], &["aps"]); + let evidence = evidence( + vec![dom("ad-atf-0")], + vec![gpt_slot("/123/news/atf", "ad-atf-0", &[(300, 250)])], + Vec::new(), + ); + + let result = compare_page_evidence( + &[expected], + &evidence, + RuntimeGateSummary::unknown_allowed(), + ); + + assert_eq!( + result.slots[0].status, + SlotStatus::Confirmed, + "missing APS does not flip status" + ); + assert!(result.slots[0].warnings.is_empty()); + assert!( + !result.strict_failed(), + "provider warning alone must not fail strict" + ); + } +} diff --git a/crates/trusted-server-cli/src/ad_templates/expected.rs b/crates/trusted-server-cli/src/ad_templates/expected.rs new file mode 100644 index 000000000..9392963ff --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/expected.rs @@ -0,0 +1,336 @@ +//! Pure expected-slot projection from the runtime creative-opportunity matcher. +//! +//! This module owns path/URL normalization and converts the slots matched by +//! [`match_slots`] into stable, owned [`ExpectedSlot`] records for output and +//! browser-evidence comparison. It must not duplicate glob-matching semantics. + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{CreativeOpportunitiesConfig, match_slots}; +use url::Url; + +/// The expected slots for a single page path, in configured slot order. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedSlots { + /// The page path the slots were matched against. + pub path: String, + /// Matched slots projected into stable records, in configured order. + pub slots: Vec, +} + +/// A single configured slot expected to appear for a page path. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedSlot { + /// The slot identifier. + pub id: String, + /// Resolved HTML `div` element ID (override or the slot id). + pub div_id: String, + /// Resolved GAM unit path: the rendered `gam_unit_path` template (or + /// `//` when the slot has none). + /// + /// `None` only for manually constructed comparison fixtures. Projection + /// omits a slot when the runtime cannot render it for this path. + pub gam_unit_path: Option, + /// Configured ad formats. + pub formats: Vec, + /// Configured provider names, in `aps`, `prebid` order. + pub providers: Vec, + /// Glob patterns configured for this slot. + pub page_patterns: Vec, +} + +/// A configured ad format as a stable width/height/media-type record. +#[derive(Debug, Clone, PartialEq)] +pub struct ExpectedFormat { + /// Creative width in pixels. + pub width: u32, + /// Creative height in pixels. + pub height: u32, + /// Configured media type. + pub media_type: MediaType, +} + +/// Projects the slots matching `path` into stable expected-slot records. +/// +/// Uses [`match_slots`] so glob semantics stay identical to the runtime, and +/// preserves configured slot order. `path` is assumed already normalized via +/// [`normalize_path_or_url`]. +/// +/// `gam_unit_path` templates are rendered against the section the runtime would +/// derive from `path` (per the config's `section_root`/`section_segment` +/// policy), so `{section}`-bearing configs project the same unit path the live +/// page requests. +// Shared projection used by the audit verifier; the static commands match slots +// directly against the runtime matcher. +#[must_use] +pub fn expected_slots_for_path(path: &str, config: &CreativeOpportunitiesConfig) -> ExpectedSlots { + let section = config.section_for_path(path); + let slots = match_slots(&config.slot, path) + .into_iter() + .filter_map(|slot| { + let gam_unit_path = slot.render_gam_unit_path(&config.gam_network_id, §ion)?; + Some(ExpectedSlot { + id: slot.id.clone(), + div_id: slot.resolved_div_id().to_string(), + gam_unit_path: Some(gam_unit_path), + formats: slot + .formats + .iter() + .map(|format| ExpectedFormat { + width: format.width, + height: format.height, + media_type: format.media_type.clone(), + }) + .collect(), + providers: provider_names(slot), + page_patterns: slot.page_patterns.clone(), + }) + }) + .collect(); + + ExpectedSlots { + path: path.to_string(), + slots, + } +} + +fn provider_names( + slot: &trusted_server_core::creative_opportunities::CreativeOpportunitySlot, +) -> Vec { + let mut providers = Vec::new(); + if slot.providers.aps.is_some() { + providers.push("aps".to_string()); + } + if slot.providers.prebid.is_some() { + providers.push("prebid".to_string()); + } + providers +} + +/// Normalizes a page path or full URL into a request path. +/// +/// Full `scheme://` inputs are parsed and reduced to their path; bare inputs have +/// query and fragment stripped and a leading `/` ensured. Empty paths become `/`. +/// +/// # Errors +/// +/// Returns a user-facing string when a `scheme://` input cannot be parsed as a URL. +pub fn normalize_path_or_url(input: &str) -> Result { + let path_input = input.split(['?', '#']).next().unwrap_or(input); + let scheme_prefix = path_input.split_once("://").map(|(scheme, _)| scheme); + let has_url_scheme = scheme_prefix.is_some_and(|scheme| { + let mut chars = scheme.chars(); + chars.next().is_some_and(|ch| ch.is_ascii_alphabetic()) + && chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '+' | '-' | '.')) + }); + if has_url_scheme { + let url = Url::parse(input).map_err(|err| format!("invalid URL `{input}`: {err}"))?; + let path = url.path(); + return Ok(if path.is_empty() { + "/".to_string() + } else { + path.to_string() + }); + } + + let base = Url::parse("https://path-normalizer.example/") + .expect("should parse static path normalization base"); + let relative = input.trim_start_matches('/'); + let normalized = base + .join(&format!("./{relative}")) + .map_err(|error| format!("invalid path `{input}`: {error}"))?; + Ok(normalized.path().to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn creative_config_with_slots(patterns: &[&str]) -> CreativeOpportunitiesConfig { + let page_patterns = patterns + .iter() + .map(|pattern| format!("\"{pattern}\"")) + .collect::>() + .join(", "); + let toml = format!( + "gam_network_id = \"123\"\n\ + \n\ + [[slot]]\n\ + id = \"atf\"\n\ + gam_unit_path = \"/123/news/atf\"\n\ + div_id = \"ad-atf-\"\n\ + page_patterns = [{page_patterns}]\n\ + formats = [{{ width = 300, height = 250 }}]\n\ + \n\ + [slot.providers.prebid]\n\ + bidders = {{}}\n" + ); + let mut config = toml::from_str::(&toml) + .expect("should deserialize creative opportunities config"); + config.compile_slots(); + config + } + + #[test] + fn expected_slots_use_runtime_matcher_and_config_order() { + let config = creative_config_with_slots(&["/news/*", "/"]); + let expected = expected_slots_for_path("/news/story", &config); + + assert_eq!(expected.path, "/news/story"); + assert_eq!( + expected + .slots + .iter() + .map(|slot| slot.id.as_str()) + .collect::>(), + ["atf"] + ); + assert_eq!(expected.slots[0].div_id, "ad-atf-"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/123/news/atf") + ); + assert_eq!(expected.slots[0].providers, ["prebid"]); + assert_eq!( + expected.slots[0].formats, + vec![ExpectedFormat { + width: 300, + height: 250, + media_type: MediaType::Banner, + }] + ); + } + + #[test] + fn expected_slots_default_resolution_without_overrides() { + let toml = "gam_network_id = \"42\"\n\ + \n\ + [[slot]]\n\ + id = \"footer\"\n\ + page_patterns = [\"/\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + let expected = expected_slots_for_path("/", &config); + assert_eq!(expected.slots[0].div_id, "footer"); + assert_eq!( + expected.slots[0].gam_unit_path.as_deref(), + Some("/42/footer") + ); + assert!(expected.slots[0].providers.is_empty()); + } + + #[test] + fn expected_slots_render_section_templates_per_path() { + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{network_id}/example/{section}\"\n\ + page_patterns = [\"/\", \"/news\", \"/news/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + // A path with a section segment renders that segment. + assert_eq!( + expected_slots_for_path("/news/story", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/news"), + "a section template should render the path's section" + ); + // The site root falls back to the configured section_root. + assert_eq!( + expected_slots_for_path("/", &config).slots[0] + .gam_unit_path + .as_deref(), + Some("/99999/example/homepage"), + "the root path should render section_root" + ); + } + + #[test] + fn expected_slots_omit_dynamic_template_the_runtime_cannot_render() { + // A `{section}` template that renders past GAM's 100-byte unit-path + // limit. The runtime omits this slot for the request path, so diagnostics + // must not match it against a truncated or otherwise different path. + let toml = "gam_network_id = \"99999\"\n\ + section_root = \"homepage\"\n\ + \n\ + [[slot]]\n\ + id = \"ad-header-0\"\n\ + gam_unit_path = \"/{section}/{section}\"\n\ + page_patterns = [\"/*\"]\n\ + formats = [{ width = 728, height = 90 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + + let long_path = format!("/{}", "a".repeat(60)); + let expected = expected_slots_for_path(&long_path, &config); + + assert!( + expected.slots.is_empty(), + "the runtime omits an over-limit dynamic slot on this path" + ); + } + + #[test] + fn normalize_path_or_url_strips_query_and_fragment() { + assert_eq!( + normalize_path_or_url("https://www.example.com/news/story?x=1#top") + .expect("should normalize"), + "/news/story" + ); + assert_eq!( + normalize_path_or_url("news/story?x=1").expect("should normalize"), + "/news/story" + ); + } + + #[test] + fn normalize_path_or_url_roots_empty_input() { + assert_eq!( + normalize_path_or_url("https://www.example.com").expect("should normalize"), + "/" + ); + assert_eq!(normalize_path_or_url("").expect("should normalize"), "/"); + } + + #[test] + fn normalize_path_or_url_uses_identical_url_rules_for_bare_paths() { + assert_eq!( + normalize_path_or_url("/a/../b").expect("should normalize bare dot segment"), + "/b" + ); + assert_eq!( + normalize_path_or_url("https://example.com/a/../b") + .expect("should normalize URL dot segment"), + "/b" + ); + assert_eq!( + normalize_path_or_url("/a b").expect("should encode bare path"), + "/a%20b" + ); + assert_eq!( + normalize_path_or_url("/r?to=https://example.com") + .expect("query URL should not change input classification"), + "/r" + ); + assert_eq!( + normalize_path_or_url("/news:latest").expect("colon should stay in bare path"), + "/news:latest", + "a colon in the first segment must not be parsed as a URL scheme" + ); + assert_eq!( + normalize_path_or_url("https://example.com/news:latest") + .expect("colon should stay in URL path"), + "/news:latest", + "bare and absolute forms should normalize identically" + ); + } +} diff --git a/crates/trusted-server-cli/src/ad_templates/mod.rs b/crates/trusted-server-cli/src/ad_templates/mod.rs new file mode 100644 index 000000000..3c26bf121 --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/mod.rs @@ -0,0 +1,7 @@ +//! Pure, host-only ad-template CLI logic shared by the static `ts config +//! ad-templates ...` commands and the browser-backed `ts audit ad-templates +//! verify` command. + +pub mod compare; +pub mod expected; +pub mod output; diff --git a/crates/trusted-server-cli/src/ad_templates/output.rs b/crates/trusted-server-cli/src/ad_templates/output.rs new file mode 100644 index 000000000..e12c9eebc --- /dev/null +++ b/crates/trusted-server-cli/src/ad_templates/output.rs @@ -0,0 +1,480 @@ +//! Stable, serializable output model for ad-template diagnostics. +//! +//! These types mirror the `--json` contract in +//! `docs/superpowers/specs/2026-06-26-server-side-ad-template-cli-design.md` §8. +//! Field names and declaration order are load-bearing: `serde` serializes struct +//! fields in declaration order, so the order here must match the spec examples. +//! +//! The model is consumed by the `ts audit ad-templates verify` orchestrator, +//! which assembles these wire types from the URL/gate context and comparison result. + +use std::borrow::Cow; + +use serde::{Deserialize, Serialize}; + +use trusted_server_core::creative_opportunities::RuntimeAdStackExpected; + +/// Escapes control characters in page-controlled text bound for a terminal. +/// +/// Page titles and collector warning messages are attacker-controlled: an +/// audited page can put ANSI/OSC escape sequences in `document.title` and drive +/// the operator's terminal (cursor movement, clipboard writes, forged output) +/// when the value is printed verbatim. Every C0 control (including ESC), DEL, +/// and the C1 range are rendered as `\u{XXXX}` so the text stays inert. JSON +/// output is unaffected — `serde_json` escapes these already. +/// +/// Returns a borrowed `Cow` when the input needs no escaping. +#[must_use] +pub fn escape_terminal_text(value: &str) -> Cow<'_, str> { + if !value.chars().any(is_terminal_control) { + return Cow::Borrowed(value); + } + let mut escaped = String::with_capacity(value.len()); + for ch in value.chars() { + if is_terminal_control(ch) { + escaped.push_str(&format!("\\u{{{:04X}}}", ch as u32)); + } else { + escaped.push(ch); + } + } + Cow::Owned(escaped) +} + +/// Whether `ch` can act as a terminal control code (C0, DEL, or C1). +fn is_terminal_control(ch: char) -> bool { + let code = ch as u32; + code < 0x20 + || (0x7f..=0x9f).contains(&code) + || (0x202a..=0x202e).contains(&code) + || (0x2066..=0x2069).contains(&code) +} + +/// Confirmation status for a single configured slot. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SlotStatus { + /// GPT evidence matches GAM path, div, and a compatible size. + Confirmed, + /// Some evidence, but not enough to confirm. + Partial, + /// No DOM or GPT evidence confirms the slot. + Missing, + /// The checker does not support confirming this slot type. + Unconfirmable, +} + +/// JSON rendering of the runtime ad-stack expectation. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RuntimeAdStackExpectedJson { + /// The server-side ad stack is expected to run. + Yes, + /// A known gate blocks the server-side ad stack. + No, + /// Consent or another gate is unprovable. + Unknown, +} + +impl From for RuntimeAdStackExpectedJson { + fn from(value: RuntimeAdStackExpected) -> Self { + match value { + RuntimeAdStackExpected::Yes => Self::Yes, + RuntimeAdStackExpected::No => Self::No, + RuntimeAdStackExpected::Unknown => Self::Unknown, + } + } +} + +/// State of a single runtime gate. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GateState { + /// The gate passed. + Pass, + /// The gate blocked the ad stack. + Fail, + /// The gate state could not be proven. + Unknown, +} + +/// Evidence-collection phase, rendered for JSON output. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidencePhaseJson { + /// Observed during the initial page load and settle. + InitialLoad, + /// Observed only after the deterministic scroll pass. + Scroll, +} + +/// A structured warning with a stable machine code and human message. +/// +/// `Serialize` for output; `Deserialize` because the browser collector payload +/// carries warning objects decoded into the comparison input. +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct Warning { + /// Stable machine-readable code (e.g. `dom_without_gpt`). + pub code: String, + /// Human-readable message; JSON consumers must not parse this. + pub message: String, +} + +/// Top-level `--json` document for `ts audit ad-templates verify`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct VerificationReport { + /// True when no strict failure and no page-level error occurred. + pub ok: bool, + /// Whether `--strict` was set. + pub strict: bool, + /// One entry per requested URL, in input order. + pub pages: Vec, + /// Run-level warnings not attributable to a single page. + /// + /// Always empty today — every warning the verifier raises belongs to a page + /// or a slot. Kept because the JSON schema declares it, so a consumer can + /// read it unconditionally. + pub warnings: Vec, +} + +/// A single audited page result. +/// +/// `error` is declared immediately after `path` so the serialized key order +/// matches the spec §8 `navigation_failed` shape; on normal pages it is `None` +/// and skipped, leaving the runtime/gates fields in §8 order. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct PageJson { + /// The requested URL. + pub url: String, + /// The final URL after redirects, or `null` on navigation failure. + pub final_url: Option, + /// The requested URL's path. + pub requested_path: String, + /// The final path used for matching, or `null` on navigation failure. + pub path: Option, + /// Present only on a page-level collection failure. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Three-state runtime ad-stack expectation; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_ad_stack_expected: Option, + /// Per-gate evidence; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub gates: Option, + /// Number of configured slots matched for the final path; absent on error pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_slot_count: Option, + /// Per-slot verification results. + pub slots: Vec, + /// Live ad-slot evidence with no matching configured slot. + pub extra_evidence: Vec, + /// Page-level warnings. + pub warnings: Vec, +} + +/// Runtime gate states for a page, one field per spec §5.2 gate. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Gates { + /// Request method is `GET`. + pub method_get: GateState, + /// Request is a top-level navigation. + pub navigation: GateState, + /// Request is not a prefetch. + pub not_prefetch: GateState, + /// Request is not from a known bot. + pub not_bot: GateState, + /// At least one configured slot matched the final path. + pub matched_slots: GateState, + /// The `[auction].enabled` kill switch is on. + pub auction_enabled: GateState, + /// Consent allows the auction (often `unknown` for live requests). + pub consent_allows_auction: GateState, +} + +/// A single configured slot's verification result. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SlotJson { + /// The configured slot id. + pub id: String, + /// The slot's confirmation status. + pub status: SlotStatus, + /// The phase the confirming evidence was observed in. + #[serde(skip_serializing_if = "Option::is_none")] + pub phase: Option, + /// The configured shape of the slot (no `id`/`page_patterns` per §8). + pub configured: ConfiguredJson, + /// The live evidence observed for this slot. + pub evidence: SlotEvidenceJson, + /// Slot-level warnings (e.g. provider or size warnings). + pub warnings: Vec, +} + +/// The configured shape of a slot, as rendered in §8 `configured`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ConfiguredJson { + /// Resolved div element ID. + pub div_id: String, + /// Resolved GAM unit path, or `null` when a dynamic template renders past + /// GAM's unit-path byte limit for this page's section. + pub gam_unit_path: Option, + /// Configured formats. + pub formats: Vec, + /// Configured provider names. + pub providers: Vec, +} + +/// A configured format, as rendered in §8. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct FormatJson { + /// Creative width in pixels. + pub width: u32, + /// Creative height in pixels. + pub height: u32, + /// Media type string (`banner`, `video`, `native`). + pub media_type: String, +} + +/// Live evidence observed for a configured slot. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct SlotEvidenceJson { + /// The resolved DOM element ID observed, if any. + pub dom_id: Option, + /// GPT slot evidence, if any (no `phase` key per §8). + pub gpt: Option, +} + +/// GPT slot evidence, as rendered in §8 `evidence.gpt`. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct GptEvidenceJson { + /// The observed GAM ad unit path. + pub gam_unit_path: String, + /// The observed GPT slot element ID. + pub div_id: String, + /// Observed numeric sizes as `[width, height]` pairs. + pub sizes: Vec<[u32; 2]>, +} + +/// Live ad-slot evidence with no matching configured slot. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ExtraEvidenceJson { + /// Evidence kind: `dom`, `gpt`, or `aps`. + pub kind: String, + /// The phase the evidence was observed in. + pub phase: EvidencePhaseJson, + /// The DOM element ID, if any. + pub dom_id: Option, + /// The GAM unit path, if any. + pub gam_unit_path: Option, + /// Observed numeric sizes as `[width, height]` pairs. + pub sizes: Vec<[u32; 2]>, + /// Why this evidence is reported as extra. + pub reason: String, +} + +#[cfg(test)] +impl VerificationReport { + fn example_confirmed_with_extra_evidence() -> Self { + VerificationReport { + ok: true, + strict: false, + pages: vec![PageJson { + url: "https://www.example.com/news/story".to_string(), + final_url: Some("https://www.example.com/news/story".to_string()), + requested_path: "/news/story".to_string(), + path: Some("/news/story".to_string()), + error: None, + runtime_ad_stack_expected: Some(RuntimeAdStackExpectedJson::Unknown), + gates: Some(Gates { + method_get: GateState::Pass, + navigation: GateState::Pass, + not_prefetch: GateState::Pass, + not_bot: GateState::Pass, + matched_slots: GateState::Pass, + auction_enabled: GateState::Pass, + consent_allows_auction: GateState::Unknown, + }), + matched_slot_count: Some(1), + slots: vec![SlotJson { + id: "atf".to_string(), + status: SlotStatus::Confirmed, + phase: Some(EvidencePhaseJson::InitialLoad), + configured: ConfiguredJson { + div_id: "ad-atf-".to_string(), + gam_unit_path: Some("/123/news/atf".to_string()), + formats: vec![FormatJson { + width: 300, + height: 250, + media_type: "banner".to_string(), + }], + providers: vec!["aps".to_string()], + }, + evidence: SlotEvidenceJson { + dom_id: Some("ad-atf-0".to_string()), + gpt: Some(GptEvidenceJson { + gam_unit_path: "/123/news/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![[300, 250]], + }), + }, + warnings: Vec::new(), + }], + extra_evidence: vec![ExtraEvidenceJson { + kind: "gpt".to_string(), + phase: EvidencePhaseJson::InitialLoad, + dom_id: Some("ad-right-rail-0".to_string()), + gam_unit_path: Some("/123/publisher/right-rail".to_string()), + sizes: vec![[300, 250]], + reason: "no_configured_slot_matched".to_string(), + }], + warnings: vec![Warning { + code: "redirected".to_string(), + message: "navigation redirected to the final path".to_string(), + }], + }], + warnings: Vec::new(), + } + } + + fn example_navigation_failed() -> Self { + VerificationReport { + ok: false, + strict: false, + pages: vec![PageJson { + url: "https://www.example.com/broken".to_string(), + final_url: None, + requested_path: "/broken".to_string(), + path: None, + error: Some(Warning { + code: "navigation_failed".to_string(), + message: "failed to read main document navigation response".to_string(), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + }], + warnings: Vec::new(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn escape_terminal_text_passes_through_ordinary_titles() { + assert!( + matches!( + escape_terminal_text("Example News — Story"), + Cow::Borrowed(_) + ), + "text with no control characters should not allocate" + ); + assert_eq!( + escape_terminal_text("Example News — Story"), + "Example News — Story" + ); + } + + #[test] + fn escape_terminal_text_neutralizes_control_sequences() { + // ESC-based CSI/OSC sequences and a raw newline are the terminal-driving + // primitives a hostile page would put in `document.title`. + assert_eq!( + escape_terminal_text("a\u{1b}]0;pwned\u{7}b"), + "a\\u{001B}]0;pwned\\u{0007}b", + "ESC and BEL should be rendered inert" + ); + assert_eq!( + escape_terminal_text("line\nforged: ok"), + "line\\u{000A}forged: ok", + "a newline should not let a title forge an output line" + ); + assert_eq!( + escape_terminal_text("del\u{7f}c1\u{9b}"), + "del\\u{007F}c1\\u{009B}", + "DEL and the C1 range should be escaped too" + ); + assert_eq!( + escape_terminal_text("safe\u{202E}forged\u{2066}tail"), + "safe\\u{202E}forged\\u{2066}tail", + "Unicode bidi controls should be rendered inert" + ); + } + + #[test] + fn verification_json_contains_gate_state_and_extra_evidence() { + let result = VerificationReport::example_confirmed_with_extra_evidence(); + let value = serde_json::to_value(&result).expect("should serialize"); + + assert_eq!(value["ok"], true); + assert_eq!(value["pages"][0]["requested_path"], "/news/story"); + assert_eq!(value["pages"][0]["runtime_ad_stack_expected"], "unknown"); + assert_eq!( + value["pages"][0]["gates"]["consent_allows_auction"], + "unknown" + ); + assert_eq!(value["pages"][0]["slots"][0]["status"], "confirmed"); + assert_eq!( + value["pages"][0]["slots"][0]["evidence"]["gpt"]["sizes"][0][0], + 300 + ); + assert_eq!(value["pages"][0]["extra_evidence"][0]["kind"], "gpt"); + assert_eq!(value["pages"][0]["warnings"][0]["code"], "redirected"); + // `configured` excludes id/page_patterns per §8. + assert!(value["pages"][0]["slots"][0]["configured"]["id"].is_null()); + assert!(value["pages"][0]["slots"][0]["configured"]["page_patterns"].is_null()); + // `evidence.gpt` has no `phase` key per §8. + assert!(value["pages"][0]["slots"][0]["evidence"]["gpt"]["phase"].is_null()); + } + + #[test] + fn page_error_json_matches_navigation_failed_shape() { + let result = VerificationReport::example_navigation_failed(); + let value = serde_json::to_value(&result).expect("should serialize"); + let page = &value["pages"][0]; + + assert_eq!(page["error"]["code"], "navigation_failed"); + assert!(page["final_url"].is_null(), "final_url should be null"); + assert!(page["path"].is_null(), "path should be null"); + assert!( + page.get("runtime_ad_stack_expected").is_none(), + "runtime field absent on error page" + ); + assert!(page.get("gates").is_none(), "gates absent on error page"); + assert!( + page.get("matched_slot_count").is_none(), + "matched_slot_count absent on error page" + ); + assert_eq!(value["ok"], false); + } + + #[test] + fn missing_slot_json_omits_evidence_phase() { + let slot = SlotJson { + id: "missing".to_string(), + status: SlotStatus::Missing, + phase: None, + configured: ConfiguredJson { + div_id: "ad-missing-".to_string(), + gam_unit_path: Some("/123/publisher/missing".to_string()), + formats: Vec::new(), + providers: Vec::new(), + }, + evidence: SlotEvidenceJson { + dom_id: None, + gpt: None, + }, + warnings: Vec::new(), + }; + + let value = serde_json::to_value(slot).expect("should serialize missing slot"); + + assert!( + value.get("phase").is_none(), + "missing evidence should not claim an initial-load phase" + ); + } +} diff --git a/crates/trusted-server-cli/src/app_config.rs b/crates/trusted-server-cli/src/app_config.rs new file mode 100644 index 000000000..bee536146 --- /dev/null +++ b/crates/trusted-server-cli/src/app_config.rs @@ -0,0 +1,171 @@ +//! Shared effective Trusted Server app-config loading for the `ts` CLI. +//! +//! Both the static `ts config ad-templates ...` commands and the browser-backed +//! `ts audit ad-templates verify` command load the same effective app config +//! through [`load_settings`], so config-path resolution and the `EdgeZero` +//! environment overlay stay consistent across command families. + +use std::path::{Path, PathBuf}; + +use clap::Args; +use edgezero_core::app_config::{self, AppConfigLoadOptions}; +use edgezero_core::manifest::ManifestLoader; +use trusted_server_core::config::TrustedServerAppConfig; +use trusted_server_core::settings::Settings; + +/// Shared local app-config flags accepted by every config/audit ad-template command. +#[derive(Clone, Debug, Args)] +pub struct AppConfigArgs { + /// Path to `trusted-server.toml`. Defaults to `.toml` beside `edgezero.toml`. + #[arg(long)] + pub app_config: Option, + /// Path to `edgezero.toml`. + #[arg(long, default_value = "edgezero.toml")] + pub manifest: PathBuf, + /// Skip app-config environment overlay. + #[arg(long)] + pub no_env: bool, +} + +/// Effective settings plus the resolved app-config path they were loaded from. +#[derive(Debug)] +pub struct LoadedSettings { + /// The `trusted-server.toml` path the settings were loaded from. + pub app_config_path: PathBuf, + /// The deserialized effective settings. + pub settings: Settings, +} + +/// Loads the effective Trusted Server settings described by `args`. +/// +/// Resolves the app-config path from `args` (or the manifest's `.toml` +/// default), applies the `EdgeZero` environment overlay unless `no_env` is set, and +/// returns the deserialized [`Settings`]. +/// +/// # Errors +/// +/// Returns a user-facing string when the manifest cannot be loaded, has no +/// `[app].name`, or the resolved app-config file cannot be read or parsed. When an +/// explicit `--app-config` path is given and is missing, the error names that +/// exact path rather than silently falling back. +pub fn load_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, !args.no_env) +} + +/// Loads Trusted Server settings from the resolved app-config file without +/// applying environment overlays. +/// +/// Mutating commands use this path so environment-only values are never +/// persisted into the operator-owned TOML file. +/// +/// # Errors +/// +/// Returns the same path-resolution, read, and parse errors as +/// [`load_settings`]. +#[cfg(test)] +pub(crate) fn load_file_settings(args: &AppConfigArgs) -> Result { + load_settings_with_env_overlay(args, false) +} + +/// Resolves the operator-owned app-config path without deserializing settings. +/// +/// Mutating recovery commands use this when the existing config may already be +/// invalid but still needs a narrowly scoped structural repair. +/// +/// # Errors +/// +/// Returns a user-facing string when the manifest cannot be loaded or has no +/// `[app].name` and no explicit config path was supplied. +pub fn resolve_app_config_file(args: &AppConfigArgs) -> Result { + if let Some(path) = &args.app_config { + return Ok(path.clone()); + } + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + Ok(resolve_app_config_path(None, &args.manifest, &app_name)) +} + +fn load_settings_with_env_overlay( + args: &AppConfigArgs, + env_overlay: bool, +) -> Result { + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let app_name = manifest_loader.manifest().app.name.clone().ok_or_else(|| { + format!( + "{} has no [app].name; cannot resolve trusted-server.toml", + args.manifest.display() + ) + })?; + let app_config_path = + resolve_app_config_path(args.app_config.as_deref(), &args.manifest, &app_name); + + let mut opts = AppConfigLoadOptions::default(); + opts.env_overlay = env_overlay; + let app_config = app_config::deserialize_app_config_with_options::( + &app_config_path, + &app_name, + &opts, + ) + .map_err(|err| format!("failed to load {}: {err}", app_config_path.display()))?; + + Ok(LoadedSettings { + app_config_path, + settings: app_config.into_settings(), + }) +} + +fn resolve_app_config_path( + explicit: Option<&Path>, + manifest_path: &Path, + app_name: &str, +) -> PathBuf { + if let Some(path) = explicit { + return path.to_path_buf(); + } + let file_name = format!("{app_name}.toml"); + if let Some(parent) = manifest_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + parent.join(file_name) + } else { + PathBuf::from(file_name) + } +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::TempDir; + + use super::*; + + #[test] + fn explicit_missing_app_config_does_not_fall_back() { + let temp = TempDir::new().expect("should create temp dir"); + let manifest_path = temp.path().join("edgezero.toml"); + fs::write(&manifest_path, "[app]\nname = \"trusted-server\"\n") + .expect("should write manifest"); + let missing_path = temp.path().join("missing.toml"); + + let args = AppConfigArgs { + app_config: Some(missing_path.clone()), + manifest: manifest_path, + no_env: true, + }; + + let err = load_settings(&args).expect_err("should reject missing explicit config"); + assert!( + err.contains(missing_path.to_string_lossy().as_ref()), + "error should mention the explicit missing path" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js new file mode 100644 index 000000000..6938808f5 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/ad_template_collector.js @@ -0,0 +1,241 @@ +// Bounded ad-template evidence collector, injected before publisher scripts run. +// +// This body runs inside an IIFE that defines `__TS_CONFIG` (the configured div +// prefixes). It records evidence into `window.__tsAdTemplateEvidence` +// and never captures page HTML, cookies, storage, request bodies, or arbitrary DOM. +// It always calls original page functions with unchanged arguments and never +// spoofs the browser automation flag. + +const __ts_config = typeof __TS_CONFIG === "object" && __TS_CONFIG ? __TS_CONFIG : {} +const __ts_prefixes = Array.isArray(__ts_config.div_prefixes) ? __ts_config.div_prefixes : [] + +const __ts_ev = (window.__tsAdTemplateEvidence = window.__tsAdTemplateEvidence || { + dom_ids: [], + gpt_slots: [], + aps_calls: [], + warnings: [] +}) + +const __ts_phase = () => (window.__tsScrollPhase ? "scroll" : "initial_load") + +// Hard cap per evidence list so a hostile page cannot grow the store without +// bound; the page controls how many slots/elements/warnings it produces. +const __ts_max_entries = 128 +const __ts_max_string_length = 512 +const __ts_wrapped_googletags = new WeakSet() + +function __ts_text(value) { + return String(value).slice(0, __ts_max_string_length) +} + +// Truncation has to be visible: surplus configured slots classify Missing, and +// `--strict` counts that, so a silent drop is indistinguishable from real drift. +let __ts_truncated = false +function __ts_push(list, entry) { + if (list.length < __ts_max_entries) { + list.push(entry) + return + } + if (__ts_truncated) return + __ts_truncated = true + if (__ts_ev.warnings.length < __ts_max_entries) { + __ts_ev.warnings.push({ + code: "evidence_truncated", + message: "an evidence list hit the " + __ts_max_entries + "-entry cap; results are incomplete" + }) + } +} + +function __ts_warn(code, error) { + __ts_push(__ts_ev.warnings, { code, message: __ts_text(error) }) +} + +// GPT sizes reach Rust as u32 pairs, so anything non-integral (fluid slots, +// NaN, negative or fractional dimensions) must be dropped here — a single bad +// pair would fail deserialization of the whole evidence payload and discard +// every other slot's otherwise valid evidence. +function __ts_size_pair(width, height) { + if (!Number.isInteger(width) || !Number.isInteger(height)) return null + if (width < 0 || height < 0 || width > 4294967295 || height > 4294967295) return null + return [width, height] +} + +function __ts_warn_ignored_size(width, height) { + const numeric = Number.isInteger(width) && Number.isInteger(height) + const outOfRange = + numeric && (width < 0 || height < 0 || width > 4294967295 || height > 4294967295) + __ts_push(__ts_ev.warnings, { + code: outOfRange ? "size_out_of_range" : "fluid_size_ignored", + message: outOfRange ? "GPT size outside u32 range ignored" : "non-integer GPT size ignored" + }) +} + +function __ts_normalize_sizes(sizes) { + const out = [] + if (!Array.isArray(sizes)) return out + // Accept [w, h] or [[w, h], ...]; treat numeric-leading arrays as a single pair. + const pairs = typeof sizes[0] === "number" ? [sizes] : sizes + for (const size of pairs) { + if (out.length >= __ts_max_entries) break + const pair = Array.isArray(size) ? __ts_size_pair(size[0], size[1]) : null + if (pair) { + out.push(pair) + } else { + __ts_warn_ignored_size( + Array.isArray(size) ? size[0] : undefined, + Array.isArray(size) ? size[1] : undefined + ) + } + } + return out +} + +function __ts_record_define_slot(adUnitPath, sizes, divId) { + __ts_push(__ts_ev.gpt_slots, { + gam_unit_path: __ts_text(adUnitPath), + div_id: __ts_text(divId), + sizes: __ts_normalize_sizes(sizes), + phase: __ts_phase() + }) +} + +function __ts_wrap_googletag(googletag) { + if (!googletag || (typeof googletag !== "object" && typeof googletag !== "function")) { + return googletag + } + if (__ts_wrapped_googletags.has(googletag)) return googletag + __ts_wrapped_googletags.add(googletag) + // Wrap defineSlot so both direct calls and calls dispatched from the cmd queue + // are recorded (queued callbacks call this same wrapped function). + const originalDefineSlot = googletag.defineSlot + if (typeof originalDefineSlot === "function") { + try { + const descriptor = Object.getOwnPropertyDescriptor(googletag, "defineSlot") + Object.defineProperty(googletag, "defineSlot", { + configurable: true, + enumerable: descriptor ? descriptor.enumerable : true, + writable: true, + value: function (adUnitPath, sizes, divId) { + const slot = originalDefineSlot.apply(this, arguments) + try { + __ts_record_define_slot(adUnitPath, sizes, divId) + } catch (error) { + __ts_warn("define_slot_capture_failed", error) + } + return slot + } + }) + } catch (error) { + __ts_warn("define_slot_wrap_failed", error) + } + } + return googletag +} + +// Wrap an existing global or intercept a later assignment of it. +function __ts_install(name, wrap) { + if (window[name]) { + try { + wrap(window[name]) + } catch (error) { + __ts_warn(name + "_wrap_failed", error) + } + return + } + let internal + Object.defineProperty(window, name, { + configurable: true, + // A real `window.googletag` is an ordinary enumerable global; matching that + // keeps `Object.keys(window)` identical with and without the collector. + enumerable: true, + get() { + return internal + }, + set(value) { + internal = value + try { + internal = wrap(value) + } catch (error) { + __ts_warn(name + "_wrap_failed", error) + } + } + }) +} + +__ts_install("googletag", __ts_wrap_googletag) + +// On-demand DOM + getSlots scrape, invoked by the collector after settle/scroll. +window.__tsCollectAdTemplateEvidence = function () { + try { + const seen = new Set(__ts_ev.dom_ids.map((entry) => entry.dom_id)) + for (const element of document.querySelectorAll("[id]")) { + const id = __ts_text(element.id) + if (id.endsWith("-container")) continue + if (__ts_prefixes.some((prefix) => id.startsWith(prefix)) && !seen.has(id)) { + __ts_push(__ts_ev.dom_ids, { dom_id: id, phase: __ts_phase() }) + seen.add(id) + } + } + const googletag = window.googletag + if (googletag && typeof googletag.pubads === "function") { + const pubads = googletag.pubads() + const slots = typeof pubads.getSlots === "function" ? pubads.getSlots() : [] + for (const slot of slots) { + try { + const path = typeof slot.getAdUnitPath === "function" ? slot.getAdUnitPath() : "" + const divId = typeof slot.getSlotElementId === "function" ? slot.getSlotElementId() : "" + const rawSizes = typeof slot.getSizes === "function" ? slot.getSizes() : [] + const sizes = [] + for (const size of rawSizes) { + if (sizes.length >= __ts_max_entries) break + let pair = null + if ( + size && + typeof size.getWidth === "function" && + typeof size.getHeight === "function" + ) { + // A fluid GPT size answers getWidth()/getHeight() with a + // non-numeric value rather than throwing. + pair = __ts_size_pair(size.getWidth(), size.getHeight()) + } else if (Array.isArray(size)) { + pair = __ts_size_pair(size[0], size[1]) + } + if (pair) { + sizes.push(pair) + } else { + const width = + size && typeof size.getWidth === "function" + ? size.getWidth() + : Array.isArray(size) + ? size[0] + : undefined + const height = + size && typeof size.getHeight === "function" + ? size.getHeight() + : Array.isArray(size) + ? size[1] + : undefined + __ts_warn_ignored_size(width, height) + } + } + const exists = __ts_ev.gpt_slots.some( + (entry) => entry.gam_unit_path === __ts_text(path) && entry.div_id === __ts_text(divId) + ) + if (!exists) { + __ts_push(__ts_ev.gpt_slots, { + gam_unit_path: __ts_text(path), + div_id: __ts_text(divId), + sizes, + phase: __ts_phase() + }) + } + } catch (error) { + __ts_warn("gpt_scrape_failed", error) + } + } + } + } catch (error) { + __ts_warn("collect_failed", error) + } + return __ts_ev +} diff --git a/crates/trusted-server-cli/src/commands/audit/ad_templates.rs b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs new file mode 100644 index 000000000..cb11a0a0c --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/ad_templates.rs @@ -0,0 +1,1007 @@ +//! Browser-backed `ts audit ad-templates verify` orchestration. +//! +//! For each URL: collect live evidence through an [`AuditCollector`], match +//! configured slots against the **final** (post-redirect) path, evaluate the +//! runtime gate, compare evidence, and assemble the stable §8 wire result. The +//! orchestration is collector-agnostic so it is fully tested with an in-memory +//! fake collector, with no Chrome dependency. + +use std::io::{self, Write}; + +use trusted_server_core::auction::types::MediaType; +use trusted_server_core::creative_opportunities::{ + AdStackGateInput, CreativeOpportunitiesConfig, evaluate_ad_stack_gate, +}; + +use crate::ad_templates::compare::{ + BrowserAdEvidence, EvidencePhase, ExtraEvidence, RuntimeGateSummary, SlotEvidence, SlotResult, + SlotStatus as CompareStatus, compare_page_evidence, +}; +use crate::ad_templates::expected::{ExpectedSlot, expected_slots_for_path, normalize_path_or_url}; +use crate::ad_templates::output::{ + ConfiguredJson, EvidencePhaseJson, ExtraEvidenceJson, FormatJson, GateState, Gates, + GptEvidenceJson, PageJson, RuntimeAdStackExpectedJson, SlotEvidenceJson, SlotJson, SlotStatus, + VerificationReport, Warning, escape_terminal_text, +}; +use crate::commands::audit::AuditAdTemplatesVerifyArgs; +use crate::commands::audit::collector::{ + AdTemplateCollectorConfig, AuditCollector, BrowserCollectRequest, build_ad_template_init_script, +}; +use crate::run::RunOutcome; + +/// Verifies configured ad-template slots against live page evidence. +/// +/// # Errors +/// +/// Returns a user-facing string when config loading fails, or when verification +/// surfaces a page-level error or a `--strict` failure (after writing output). +pub(crate) fn run_verify(args: &AuditAdTemplatesVerifyArgs) -> Result { + args.browser.validate()?; + validate_cookie_scope(&args.urls, &args.cookies)?; + let loaded = crate::app_config::load_settings(&args.config)?; + let collector = crate::commands::audit::browser::BrowserCollector::from_opts(&args.browser); + let report = build_report( + &collector, + loaded.settings.creative_opportunities.as_ref(), + loaded.settings.auction.enabled, + &args.urls, + VerifyOptions { + strict: args.strict, + scroll: args.scroll, + allow_cross_origin_redirect: args.allow_cross_origin_redirect, + }, + &args.cookies, + )?; + + let stdout = io::stdout(); + let mut out = stdout.lock(); + if args.json { + write_json(&mut out, &report)?; + } else { + write_human(&mut out, &report)?; + } + + if report.pages.iter().any(|page| page.error.is_some()) { + Err("ad-template verification reported problems".to_string()) + } else if report.ok { + Ok(RunOutcome::Success) + } else { + Ok(RunOutcome::AssertionFailed) + } +} + +fn validate_cookie_scope(urls: &[url::Url], cookies: &[(String, String)]) -> Result<(), String> { + if cookies.is_empty() { + return Ok(()); + } + let origins: std::collections::BTreeSet = urls + .iter() + .map(|url| url.origin().ascii_serialization()) + .collect(); + if origins.len() > 1 { + return Err( + "--cookie may be used only when every verification URL has one origin; split this run so credentials are never copied to another origin" + .to_string(), + ); + } + Ok(()) +} + +/// Run-level verification switches. +#[derive(Debug, Clone, Copy)] +struct VerifyOptions { + /// Exit non-zero when a matched slot is missing or only partially confirmed. + strict: bool, + /// Perform a deterministic scroll pass after the initial settle. + scroll: bool, + /// Accept evidence from a page that redirected to a different origin. + allow_cross_origin_redirect: bool, +} + +/// Builds the verification report for `urls` using `collector`. +/// +/// `creative` is the effective `[creative_opportunities]` config (if any) and +/// `auction_enabled` is the `[auction].enabled` kill switch. +fn build_report( + collector: &dyn AuditCollector, + creative: Option<&CreativeOpportunitiesConfig>, + auction_enabled: bool, + urls: &[url::Url], + options: VerifyOptions, + cookies: &[(String, String)], +) -> Result { + let init_script = build_init_script(creative)?; + + let requests: Vec<_> = urls + .iter() + .map(|url| BrowserCollectRequest { + url: url.clone(), + init_scripts: vec![init_script.clone()], + scroll: options.scroll, + collect_ad_evidence: true, + cookies: cookies.to_vec(), + }) + .collect(); + let collected_pages = collector.collect_pages(&requests); + + let mut pages = Vec::with_capacity(urls.len()); + let mut any_error = false; + let mut any_strict_fail = false; + + for (url, collected) in urls.iter().zip(collected_pages) { + match collected { + Err(message) => { + any_error = true; + pages.push(error_page(url, &message)); + } + // Slots are matched on the *final* path, so a redirect to a + // different origin would let an unrelated site's evidence satisfy + // `--strict` — and the path-equality redirect warning would not even + // fire when the paths happen to agree. Reject unless opted in. + Ok(collected) + if !options.allow_cross_origin_redirect + && origin_changed(url, &collected.final_url) => + { + any_error = true; + pages.push(cross_origin_page(url, &collected.final_url)); + } + Ok(collected) => { + let (page, strict_failed) = build_page(url, &collected, creative, auction_enabled); + if options.strict && strict_failed { + any_strict_fail = true; + } + pages.push(page); + } + } + } + + let ok = !(any_error || (options.strict && any_strict_fail)); + Ok(VerificationReport { + ok, + strict: options.strict, + pages, + warnings: Vec::new(), + }) +} + +/// The URL without its fragment, for comparisons the server can observe. +pub(super) fn without_fragment(url: &url::Url) -> url::Url { + let mut url = url.clone(); + url.set_fragment(None); + url +} + +/// Whether navigation left the requested URL's origin (scheme, host, or port). +/// +/// A same-host default-port `http:80` to `https:443` redirect is *not* a change: +/// the host is the cookie boundary, and that upgrade is the ordinary canonical +/// redirect. Host changes, port changes, and HTTPS downgrades all are. +pub(super) fn origin_changed(requested: &url::Url, final_url: &url::Url) -> bool { + if requested.host_str() != final_url.host_str() { + return true; + } + + match (requested.scheme(), final_url.scheme()) { + ("http", "https") => { + requested.port_or_known_default() != Some(80) + || final_url.port_or_known_default() != Some(443) + } + (requested_scheme @ ("http" | "https"), final_scheme) + if requested_scheme == final_scheme => + { + requested.port_or_known_default() != final_url.port_or_known_default() + } + // Refuse HTTPS downgrades and any unexpected scheme transition. + _ => true, + } +} + +/// Builds the read-only collector init script from the configured slots. +fn build_init_script(creative: Option<&CreativeOpportunitiesConfig>) -> Result { + let config = AdTemplateCollectorConfig { + div_prefixes: creative + .map(|creative| { + creative + .slot + .iter() + .map(|slot| slot.resolved_div_id().to_string()) + .collect() + }) + .unwrap_or_default(), + }; + build_ad_template_init_script(&config) +} + +/// Assembles a successful page result, returning the wire `PageJson` and whether +/// the page would fail `--strict`. +fn build_page( + requested: &url::Url, + collected: &crate::commands::audit::collector::CollectedPage, + creative: Option<&CreativeOpportunitiesConfig>, + auction_enabled: bool, +) -> (PageJson, bool) { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + let final_url = &collected.final_url; + let final_path = normalize_path_or_url(final_url.as_str()).unwrap_or_else(|_| "/".into()); + + let expected = creative + .map(|creative| expected_slots_for_path(&final_path, creative).slots) + .unwrap_or_default(); + let matched = !expected.is_empty(); + + let gate = evaluate_ad_stack_gate(AdStackGateInput { + method_get: true, + navigation: true, + prefetch: false, + bot: false, + matched_slots: matched, + consent_allows_auction: None, + auction_enabled, + }); + + let evidence = collected.ad_evidence.clone().unwrap_or_else(empty_evidence); + let result = compare_page_evidence( + &expected, + &evidence, + RuntimeGateSummary::from_expected(gate.expected), + ); + let strict_failed = result.strict_failed(); + + let mut warnings: Vec = collected.warnings.to_vec(); + warnings.extend(evidence.warnings.iter().map(|warning| Warning { + code: format!("page_{}", warning.code), + message: warning.message.clone(), + })); + // Fragments never reach the server, so a fragment-only difference is not a + // redirect and slots match on the path either way. + if without_fragment(requested) != without_fragment(final_url) { + warnings.push(Warning { + code: "redirected".to_string(), + message: format!("navigation redirected from {requested} to {final_url}"), + }); + } + + let slots = expected + .iter() + .zip(result.slots.iter()) + .map(|(expected_slot, slot_result)| to_slot_json(expected_slot, slot_result)) + .collect(); + let extra_evidence = result.extra_evidence.iter().map(to_extra_json).collect(); + + let page = PageJson { + url: requested.to_string(), + final_url: Some(final_url.to_string()), + requested_path, + path: Some(final_path), + error: None, + runtime_ad_stack_expected: Some(RuntimeAdStackExpectedJson::from( + result.runtime_ad_stack_expected, + )), + gates: Some(to_gates(matched, auction_enabled)), + matched_slot_count: Some(expected.len()), + slots, + extra_evidence, + warnings, + }; + (page, strict_failed) +} + +/// Builds a page-level navigation-failure result (spec §8 `navigation_failed`). +fn error_page(requested: &url::Url, message: &str) -> PageJson { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + PageJson { + url: requested.to_string(), + final_url: None, + requested_path, + path: None, + error: Some(Warning { + code: "navigation_failed".to_string(), + message: message.to_string(), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + } +} + +/// Builds a page-level cross-origin-redirect refusal. +/// +/// The final URL is reported so the operator can re-run against it explicitly +/// (or pass `--allow-cross-origin-redirect`) once they have confirmed it is +/// their own property. +fn cross_origin_page(requested: &url::Url, final_url: &url::Url) -> PageJson { + let requested_path = normalize_path_or_url(requested.as_str()).unwrap_or_else(|_| "/".into()); + PageJson { + url: requested.to_string(), + final_url: Some(final_url.to_string()), + requested_path, + path: None, + error: Some(Warning { + code: "cross_origin_redirect".to_string(), + message: format!( + "navigation left the requested origin ({} -> {}); \ + evidence from another origin is not accepted as verification. \ + Re-run against the final URL, or pass --allow-cross-origin-redirect", + requested.origin().ascii_serialization(), + final_url.origin().ascii_serialization(), + ), + }), + runtime_ad_stack_expected: None, + gates: None, + matched_slot_count: None, + slots: Vec::new(), + extra_evidence: Vec::new(), + warnings: Vec::new(), + } +} + +fn empty_evidence() -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: Vec::new(), + gpt_slots: Vec::new(), + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } +} + +fn to_gates(matched: bool, auction_enabled: bool) -> Gates { + let pass_if = |cond: bool| { + if cond { + GateState::Pass + } else { + GateState::Fail + } + }; + Gates { + method_get: GateState::Pass, + navigation: GateState::Pass, + not_prefetch: GateState::Pass, + not_bot: GateState::Pass, + matched_slots: pass_if(matched), + auction_enabled: pass_if(auction_enabled), + // Live consent is not provable from a browser navigation in Phase 1. + consent_allows_auction: GateState::Unknown, + } +} + +fn to_slot_json(expected: &ExpectedSlot, result: &SlotResult) -> SlotJson { + SlotJson { + id: result.id.clone(), + status: to_status(result.status), + phase: result.phase.map(to_phase), + configured: ConfiguredJson { + div_id: expected.div_id.clone(), + gam_unit_path: expected.gam_unit_path.clone(), + formats: expected + .formats + .iter() + .map(|format| FormatJson { + width: format.width, + height: format.height, + media_type: media_type_label(&format.media_type).to_string(), + }) + .collect(), + providers: expected.providers.clone(), + }, + evidence: to_slot_evidence(&result.evidence), + warnings: result.warnings.clone(), + } +} + +fn to_slot_evidence(evidence: &SlotEvidence) -> SlotEvidenceJson { + SlotEvidenceJson { + dom_id: evidence.dom_id.clone(), + gpt: evidence.gpt.as_ref().map(|gpt| GptEvidenceJson { + gam_unit_path: gpt.gam_unit_path.clone(), + div_id: gpt.div_id.clone(), + sizes: gpt.sizes.iter().map(|&(w, h)| [w, h]).collect(), + }), + } +} + +fn to_extra_json(extra: &ExtraEvidence) -> ExtraEvidenceJson { + ExtraEvidenceJson { + kind: extra.kind.clone(), + phase: to_phase(extra.phase), + dom_id: extra.dom_id.clone(), + gam_unit_path: extra.gam_unit_path.clone(), + sizes: extra.sizes.iter().map(|&(w, h)| [w, h]).collect(), + reason: extra.reason.clone(), + } +} + +fn to_status(status: CompareStatus) -> SlotStatus { + match status { + CompareStatus::Confirmed => SlotStatus::Confirmed, + CompareStatus::Partial => SlotStatus::Partial, + CompareStatus::Missing => SlotStatus::Missing, + CompareStatus::Unconfirmable => SlotStatus::Unconfirmable, + } +} + +fn to_phase(phase: EvidencePhase) -> EvidencePhaseJson { + match phase { + EvidencePhase::InitialLoad => EvidencePhaseJson::InitialLoad, + EvidencePhase::Scroll => EvidencePhaseJson::Scroll, + } +} + +fn media_type_label(media_type: &MediaType) -> &'static str { + match media_type { + MediaType::Banner => "banner", + MediaType::Video => "video", + MediaType::Native => "native", + } +} + +fn write_json(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { + let json = serde_json::to_string_pretty(report) + .map_err(|error| format!("failed to serialize verification report: {error}"))?; + writeln!(out, "{json}").map_err(write_err) +} + +fn write_human(out: &mut dyn Write, report: &VerificationReport) -> Result<(), String> { + // Warning codes and messages can originate in the audited page (the + // collector forwards `String(error)` from page scripts), so escape control + // characters before writing them to the operator's terminal. + let write_warning = |out: &mut dyn Write, indent: &str, warning: &Warning| { + writeln!( + out, + "{indent}warning [{}]: {}", + escape_terminal_text(&warning.code), + escape_terminal_text(&warning.message) + ) + .map_err(write_err) + }; + + for warning in &report.warnings { + write_warning(out, "", warning)?; + } + for page in &report.pages { + writeln!(out, "url: {}", escape_terminal_text(&page.url)).map_err(write_err)?; + if let Some(error) = &page.error { + writeln!( + out, + " error [{}]: {}", + escape_terminal_text(&error.code), + escape_terminal_text(&error.message) + ) + .map_err(write_err)?; + continue; + } + if let Some(path) = &page.path { + writeln!(out, " path: {}", escape_terminal_text(path)).map_err(write_err)?; + } + if let Some(expected) = page.runtime_ad_stack_expected { + writeln!(out, " runtime ad stack: {}", runtime_label(expected)).map_err(write_err)?; + } + if let Some(count) = page.matched_slot_count { + writeln!(out, " matched slots: {count}").map_err(write_err)?; + } + if let Some(gates) = &page.gates { + writeln!(out, " gates: {}", gates_label(gates)).map_err(write_err)?; + } + for slot in &page.slots { + writeln!( + out, + " slot {}: {}", + escape_terminal_text(&slot.id), + status_label(slot.status) + ) + .map_err(write_err)?; + for warning in &slot.warnings { + write_warning(out, " ", warning)?; + } + } + for extra in &page.extra_evidence { + writeln!( + out, + " extra {} evidence: div={} gam={} sizes={:?} ({})", + escape_terminal_text(&extra.kind), + escape_terminal_text(extra.dom_id.as_deref().unwrap_or("-")), + escape_terminal_text(extra.gam_unit_path.as_deref().unwrap_or("-")), + extra.sizes, + escape_terminal_text(&extra.reason), + ) + .map_err(write_err)?; + } + for warning in &page.warnings { + write_warning(out, " ", warning)?; + } + } + writeln!(out, "ok: {}", report.ok).map_err(write_err) +} + +fn status_label(status: SlotStatus) -> &'static str { + match status { + SlotStatus::Confirmed => "confirmed", + SlotStatus::Partial => "partial", + SlotStatus::Missing => "missing", + SlotStatus::Unconfirmable => "unconfirmable", + } +} + +fn runtime_label(expected: RuntimeAdStackExpectedJson) -> &'static str { + match expected { + RuntimeAdStackExpectedJson::Yes => "yes", + RuntimeAdStackExpectedJson::No => "no", + RuntimeAdStackExpectedJson::Unknown => "unknown", + } +} + +fn gate_label(gate: GateState) -> &'static str { + match gate { + GateState::Pass => "pass", + GateState::Fail => "fail", + GateState::Unknown => "unknown", + } +} + +fn gates_label(gates: &Gates) -> String { + format!( + "method_get={} navigation={} not_prefetch={} not_bot={} matched_slots={} auction_enabled={} consent={}", + gate_label(gates.method_get), + gate_label(gates.navigation), + gate_label(gates.not_prefetch), + gate_label(gates.not_bot), + gate_label(gates.matched_slots), + gate_label(gates.auction_enabled), + gate_label(gates.consent_allows_auction), + ) +} + +#[allow( + clippy::needless_pass_by_value, + reason = "used as a map_err fn that receives io::Error by value" +)] +fn write_err(error: io::Error) -> String { + format!("failed to write command output: {error}") +} + +#[cfg(test)] +mod tests { + use std::cell::Cell; + use std::collections::HashMap; + + use super::*; + use crate::ad_templates::compare::{DomEvidence, GptSlotEvidence}; + use crate::commands::audit::collector::CollectedPage; + + struct FakeCollector { + pages: HashMap>, + batch_calls: Cell, + } + + impl FakeCollector { + fn page(requested: &str, final_url: &str, evidence: BrowserAdEvidence) -> Self { + let mut pages = HashMap::new(); + pages.insert( + requested.to_string(), + Ok(CollectedPage { + final_url: url::Url::parse(final_url).expect("should parse final URL"), + title: String::new(), + script_count: 0, + resource_count: 0, + warnings: Vec::new(), + ad_evidence: Some(evidence), + }), + ); + Self { + pages, + batch_calls: Cell::new(0), + } + } + + fn with_error(mut self, requested: &str, message: &str) -> Self { + self.pages + .insert(requested.to_string(), Err(message.to_string())); + self + } + } + + impl AuditCollector for FakeCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.pages + .get(request.url.as_str()) + .cloned() + .unwrap_or_else(|| Err(format!("no fake page for {}", request.url))) + } + + fn collect_pages( + &self, + requests: &[BrowserCollectRequest], + ) -> Vec> { + self.batch_calls.set(self.batch_calls.get() + 1); + requests + .iter() + .cloned() + .map(|request| self.collect_page(request)) + .collect() + } + } + + fn news_config() -> CreativeOpportunitiesConfig { + let toml = "gam_network_id = \"123\"\n\ + \n\ + [[slot]]\n\ + id = \"atf\"\n\ + gam_unit_path = \"/123/news/atf\"\n\ + div_id = \"ad-atf-\"\n\ + page_patterns = [\"/news/*\"]\n\ + formats = [{ width = 300, height = 250 }]\n"; + let mut config = + toml::from_str::(toml).expect("should deserialize"); + config.compile_slots(); + config + } + + fn confirmed_news_evidence() -> BrowserAdEvidence { + BrowserAdEvidence { + dom_ids: vec![DomEvidence { + dom_id: "ad-atf-0".to_string(), + phase: EvidencePhase::InitialLoad, + }], + gpt_slots: vec![GptSlotEvidence { + gam_unit_path: "/123/news/atf".to_string(), + div_id: "ad-atf-0".to_string(), + sizes: vec![(300, 250)], + phase: EvidencePhase::InitialLoad, + }], + aps_calls: Vec::new(), + page_bids: Vec::new(), + warnings: Vec::new(), + } + } + + fn report_for( + collector: &dyn AuditCollector, + auction_enabled: bool, + strict: bool, + urls: &[&str], + ) -> VerificationReport { + report_for_with_options( + collector, + auction_enabled, + urls, + VerifyOptions { + strict, + scroll: false, + allow_cross_origin_redirect: false, + }, + ) + } + + fn report_for_with_options( + collector: &dyn AuditCollector, + auction_enabled: bool, + urls: &[&str], + options: VerifyOptions, + ) -> VerificationReport { + let config = news_config(); + let parsed: Vec = urls + .iter() + .map(|url| url::Url::parse(url).expect("should parse URL")) + .collect(); + build_report( + collector, + Some(&config), + auction_enabled, + &parsed, + options, + &[], + ) + .expect("typed collector configuration should serialize") + } + + #[test] + fn verify_uses_final_url_for_matching_after_redirect() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for(&collector, true, false, &["https://www.example.com/"]); + let json = serde_json::to_value(&report).expect("should serialize"); + + assert_eq!(json["pages"][0]["path"], "/news/story"); + assert_eq!(json["pages"][0]["slots"][0]["status"], "confirmed"); + let warnings = json["pages"][0]["warnings"] + .as_array() + .expect("should have warnings array"); + assert!( + warnings.iter().any(|w| w["code"] == "redirected"), + "redirect should emit a `redirected` warning" + ); + } + + #[test] + fn cross_origin_redirect_is_rejected_even_when_paths_match() { + // Same path on a different origin: the redirect warning would not fire, + // so without the origin check this unrelated page's evidence would + // satisfy --strict. + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://impostor.example.net/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["https://www.example.com/news/story"], + ); + + assert!(!report.ok, "a cross-origin redirect must not report ok"); + let json = serde_json::to_value(&report).expect("should serialize"); + assert_eq!(json["pages"][0]["error"]["code"], "cross_origin_redirect"); + assert!( + json["pages"][0]["slots"] + .as_array() + .expect("should have slots array") + .is_empty(), + "off-origin evidence must not be reported as slot verification" + ); + } + + #[test] + fn cross_origin_redirect_is_accepted_with_explicit_opt_in() { + let collector = FakeCollector::page( + "https://example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for_with_options( + &collector, + true, + &["https://example.com/news/story"], + VerifyOptions { + strict: true, + scroll: false, + allow_cross_origin_redirect: true, + }, + ); + + assert!( + report.ok, + "an opted-in apex -> www redirect should verify normally" + ); + assert_eq!(report.pages[0].matched_slot_count, Some(1)); + } + + #[test] + fn same_origin_path_redirect_still_verifies() { + let collector = FakeCollector::page( + "https://www.example.com/", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for(&collector, true, true, &["https://www.example.com/"]); + + assert!( + report.ok, + "a same-origin redirect should still be verified, not refused" + ); + } + + #[test] + fn same_host_http_to_https_upgrade_is_accepted() { + let collector = FakeCollector::page( + "http://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["http://www.example.com/news/story"], + ); + + assert!(report.ok, "a default-port HTTPS upgrade should be accepted"); + } + + #[test] + fn downgrade_and_port_changes_are_rejected() { + for (requested, final_url) in [ + ( + "https://www.example.com/news/story", + "http://www.example.com/news/story", + ), + ( + "https://www.example.com:8443/news/story", + "https://www.example.com:9443/news/story", + ), + ( + "http://www.example.com:8080/news/story", + "https://www.example.com:8443/news/story", + ), + ] { + let collector = FakeCollector::page(requested, final_url, confirmed_news_evidence()); + let report = report_for(&collector, true, true, &[requested]); + assert!(!report.ok, "redirect {requested} -> {final_url} must fail"); + } + } + + #[test] + fn confirmed_page_is_ok_in_default_mode() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ); + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + + assert!(report.ok, "confirmed page should be ok"); + assert_eq!(report.pages[0].matched_slot_count, Some(1)); + } + + #[test] + fn verifier_surfaces_injected_collector_warnings() { + let mut evidence = confirmed_news_evidence(); + evidence.warnings.push(Warning { + code: "fluid_size_ignored".to_string(), + message: "a fluid size could not be compared".to_string(), + }); + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + evidence, + ); + + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + + assert!( + report.pages[0] + .warnings + .iter() + .any(|warning| warning.code == "page_fluid_size_ignored"), + "collector warning should be visible in the page report" + ); + } + + #[test] + fn human_output_includes_runtime_and_extra_evidence_diagnostics() { + let mut evidence = confirmed_news_evidence(); + evidence.gpt_slots.push(GptSlotEvidence { + gam_unit_path: "/123/publisher/extra".to_string(), + div_id: "ad-extra-0".to_string(), + sizes: vec![(728, 90)], + phase: EvidencePhase::InitialLoad, + }); + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + evidence, + ); + let report = report_for( + &collector, + true, + false, + &["https://www.example.com/news/story"], + ); + let mut output = Vec::new(); + + write_human(&mut output, &report).expect("should write human report"); + let output = String::from_utf8(output).expect("should be UTF-8 output"); + + assert!(output.contains("runtime ad stack: unknown")); + assert!(output.contains("matched slots: 1")); + assert!(output.contains("gates: method_get=pass")); + assert!(output.contains("extra gpt evidence")); + } + + #[test] + fn strict_missing_slot_fails() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + empty_evidence(), + ); + let report = report_for( + &collector, + true, + true, + &["https://www.example.com/news/story"], + ); + + assert!( + !report.ok, + "strict mode with a missing slot should not be ok" + ); + } + + #[test] + fn auction_disabled_skips_strict_missing_failure() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + empty_evidence(), + ); + // auction disabled -> runtime expected No -> strict does not fail on missing. + let report = report_for( + &collector, + false, + true, + &["https://www.example.com/news/story"], + ); + + assert!( + report.ok, + "missing slot must not fail strict when auction is disabled" + ); + assert_eq!( + report.pages[0].runtime_ad_stack_expected, + Some(RuntimeAdStackExpectedJson::No) + ); + } + + #[test] + fn multi_url_page_error_sets_ok_false() { + let collector = FakeCollector::page( + "https://www.example.com/news/story", + "https://www.example.com/news/story", + confirmed_news_evidence(), + ) + .with_error("https://www.example.com/broken", "navigation failed"); + let report = report_for( + &collector, + true, + false, + &[ + "https://www.example.com/news/story", + "https://www.example.com/broken", + ], + ); + + assert!(!report.ok, "a page-level error sets ok=false"); + assert_eq!( + collector.batch_calls.get(), + 1, + "all verifier URLs should use one collector batch" + ); + let json = serde_json::to_value(&report).expect("should serialize"); + assert_eq!(json["pages"][1]["error"]["code"], "navigation_failed"); + assert!(json["pages"][1]["final_url"].is_null()); + } + + #[test] + fn supplied_cookies_are_rejected_for_multiple_origins() { + let urls = [ + url::Url::parse("https://a.example/x").expect("should parse first URL"), + url::Url::parse("https://b.example/y").expect("should parse second URL"), + ]; + + let error = validate_cookie_scope(&urls, &[("session".to_string(), "secret".to_string())]) + .expect_err("should not replicate one cookie across origins"); + + assert!( + error.contains("one origin"), + "the refusal should explain cookie scope, got {error}" + ); + } + + #[test] + fn supplied_cookies_are_allowed_for_same_origin_urls() { + let urls = [ + url::Url::parse("https://a.example/x").expect("should parse first URL"), + url::Url::parse("https://a.example/y").expect("should parse second URL"), + ]; + + validate_cookie_scope(&urls, &[("session".to_string(), "secret".to_string())]) + .expect("same-origin URLs share the intended cookie scope"); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/browser.rs b/crates/trusted-server-cli/src/commands/audit/browser.rs new file mode 100644 index 000000000..b6bbdacd5 --- /dev/null +++ b/crates/trusted-server-cli/src/commands/audit/browser.rs @@ -0,0 +1,1189 @@ +//! Chrome/Chromium-backed implementation of [`AuditCollector`] using +//! `chromiumoxide` (CDP). +//! +//! The collector installs optional pre-navigation init scripts, sets any +//! operator-supplied cookies, navigates, waits for the page to settle, optionally +//! scrolls, and reads back a bounded set of evidence. It never *captures* page +//! HTML, cookies, or storage; supplied cookies are only *sent* to carry an +//! existing session past origin gates. + +use std::time::Duration; + +use chromiumoxide::browser::{Browser, BrowserConfig}; +use chromiumoxide::cdp::browser_protocol::network::CookieParam; +use chromiumoxide::handler::viewport::Viewport; +use chromiumoxide::page::Page; +use futures::StreamExt as _; + +use crate::ad_templates::compare::BrowserAdEvidence; +use crate::ad_templates::output::Warning; +use crate::commands::audit::collector::{ + AuditCollector, BrowserCollectRequest, BrowserOpts, BrowserProfile, CollectedPage, +}; + +/// Candidate Chrome/Chromium executable names searched on `PATH`. +pub(crate) const CHROME_NAMES: &[&str] = &[ + "google-chrome", + "google-chrome-stable", + "chromium", + "chromium-browser", + "chrome", + "Google Chrome", + "Google Chrome for Testing", +]; + +/// Poll interval while waiting for the page network to settle, in milliseconds. +const SETTLE_POLL_MS: u64 = 250; +/// Hard cap on page navigation so a stalled load cannot hang the audit. +const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); +/// Bound for each CDP operation after navigation. +const CDP_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); +/// Hard cap per decoded evidence list, so a hostile page cannot inflate CLI +/// memory. +/// +/// Must equal `__ts_max_entries` in `ad_template_collector.js`. The collector +/// already caps each list, but the evidence object lives on `window`, so a page +/// that appends to it directly is bounded here instead. Anything the collector +/// itself dropped is reported as an `evidence_truncated` warning. +const MAX_EVIDENCE_ENTRIES: usize = 128; +/// Hard cap on the UTF-8 JSON payload before CDP transfers it back to Rust. +const MAX_EVIDENCE_PAYLOAD_BYTES: usize = 1024 * 1024; +/// Hard cap on browser teardown so a wedged Chrome cannot hang the audit. +const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); +/// Default quiet window (no new resources) marking the page settled. +const DEFAULT_SETTLE_QUIET_MS: u64 = 750; +/// Default hard cap on settling so slow/ad-heavy pages still terminate. +const DEFAULT_SETTLE_MAX_MS: u64 = 10_000; + +/// Page-settle timing thresholds. +#[derive(Debug, Clone, Copy)] +struct SettleConfig { + /// Quiet window with no new resources marking the page settled. + quiet: Duration, + /// Hard cap on total settle time. + max: Duration, +} + +/// Immutable browser/session settings shared by every URL in one audit batch. +struct BrowserSessionOptions<'a> { + chrome: &'a std::path::Path, + profile_dir: &'a std::path::Path, + settle: SettleConfig, + accept_invalid_certs: bool, + headful: bool, + assume_consent: bool, + proxy: Option<&'a str>, + profile: BrowserProfile, +} + +/// A `chromiumoxide`-backed page collector launching a local Chrome/Chromium. +#[derive(Debug, Clone)] +pub struct BrowserCollector { + /// Explicit Chrome/Chromium executable override (else `$CHROME`, else auto-detect). + chrome: Option, + /// Quiet window marking the page settled. + settle_quiet: Duration, + /// Hard cap on settling. + settle_max: Duration, + /// Navigate to origins with invalid TLS certificates (dangerous opt-in). + accept_invalid_certs: bool, + /// Run visible Chrome rather than new headless Chrome. + headful: bool, + /// Install the standard consent API stub before publisher scripts. + assume_consent: bool, + /// Optional browser proxy endpoint. + proxy: Option, + /// Device viewport/user-agent profile. + profile: BrowserProfile, +} + +impl Default for BrowserCollector { + fn default() -> Self { + Self::new() + } +} + +impl BrowserCollector { + /// Creates a collector with default tuning and auto-detected Chrome. + #[must_use] + pub fn new() -> Self { + Self { + chrome: None, + settle_quiet: Duration::from_millis(DEFAULT_SETTLE_QUIET_MS), + settle_max: Duration::from_millis(DEFAULT_SETTLE_MAX_MS), + accept_invalid_certs: false, + headful: false, + assume_consent: true, + proxy: None, + profile: BrowserProfile::Desktop, + } + } + + /// Creates a collector from operator-supplied browser options. + #[must_use] + pub fn from_opts(opts: &BrowserOpts) -> Self { + Self { + chrome: opts.chrome.clone(), + settle_quiet: Duration::from_millis(opts.settle_quiet_ms), + settle_max: Duration::from_millis(opts.settle_max_ms), + accept_invalid_certs: opts.danger_accept_invalid_certs, + headful: opts.headful, + assume_consent: !opts.no_assume_consent, + proxy: opts.browser_proxy.clone(), + profile: opts.profile, + } + } +} + +/// Pre-document consent behavior shared with the generation crawler. +pub(crate) const CONSENT_STUB_SCRIPT: &str = include_str!("consent_stub.js"); + +/// Shared browser launch inputs used by both audit collectors. +pub(crate) struct BrowserLaunchOptions<'a> { + pub(crate) chrome: &'a std::path::Path, + pub(crate) profile_dir: &'a std::path::Path, + pub(crate) headful: bool, + pub(crate) proxy: Option<&'a str>, + pub(crate) accept_invalid_certs: bool, + pub(crate) viewport: Viewport, + pub(crate) user_agent: Option<&'a str>, +} + +/// Builds the common Chrome configuration for all browser-backed audits. +pub(crate) fn build_browser_config( + options: BrowserLaunchOptions<'_>, +) -> Result { + let mut builder = BrowserConfig::builder() + .chrome_executable(options.chrome) + .user_data_dir(options.profile_dir); + if !options.accept_invalid_certs { + builder = builder.respect_https_errors(); + } + if let Some(proxy) = options.proxy { + let endpoint = if proxy.contains("://") { + proxy.to_string() + } else { + format!("http://{proxy}") + }; + builder = builder + .arg(("proxy-server", endpoint.as_str())) + .arg(("proxy-bypass-list", "<-loopback>")); + } + builder = if options.headful { + builder.with_head() + } else { + builder.new_headless_mode() + }; + builder = builder + .window_size(options.viewport.width, options.viewport.height) + .viewport(options.viewport); + if let Some(user_agent) = options.user_agent { + builder = builder.arg(("user-agent", user_agent)); + } + builder + .build() + .map_err(|error| format!("failed to build browser config: {error}")) +} + +fn browser_profile(profile: BrowserProfile) -> (Viewport, Option<&'static str>) { + match profile { + BrowserProfile::Desktop => ( + Viewport { + width: 1280, + height: 800, + device_scale_factor: Some(1.0), + emulating_mobile: false, + is_landscape: true, + has_touch: false, + }, + None, + ), + BrowserProfile::Mobile => ( + Viewport { + width: 390, + height: 844, + device_scale_factor: Some(3.0), + emulating_mobile: true, + is_landscape: false, + has_touch: true, + }, + Some( + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) \ + AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1", + ), + ), + } +} + +/// Resolves the Chrome/Chromium executable to launch. +/// +/// Precedence: explicit `--chrome` override, then the `CHROME` environment +/// variable, then auto-detection on `PATH` and standard install locations. +pub(crate) fn resolve_chrome( + override_path: Option<&std::path::Path>, +) -> Result { + if let Some(path) = override_path { + return if path.is_file() { + Ok(path.to_path_buf()) + } else { + Err(format!( + "--chrome path does not point to a file: {}", + path.display() + )) + }; + } + if let Ok(env_path) = std::env::var("CHROME") { + let path = std::path::PathBuf::from(&env_path); + return if path.is_file() { + Ok(path) + } else { + Err(format!("CHROME={env_path} does not point to a file")) + }; + } + find_chrome() +} + +/// Builds a host-only cookie that applies to every path on `url`'s host. +/// +/// Scoped by origin rather than by the full URL: only the origin is load-bearing +/// for a host-only cookie, and a full URL would carry the path, query, and any +/// `user:password@` into CDP and into this function's error message. +pub(crate) fn host_cookie(name: &str, value: &str, url: &url::Url) -> Result { + let origin = url.origin(); + if !origin.is_tuple() { + return Err(format!( + "cannot scope cookie `{name}` because the audited URL has no host" + )); + } + let mut cookie = CookieParam::new(name.to_string(), value.to_string()); + cookie.url = Some(origin.ascii_serialization()); + cookie.path = Some("/".to_string()); + cookie.secure = Some(url.scheme() == "https"); + Ok(cookie) +} + +fn format_cookie_install_error(name: &str, _error: impl std::fmt::Display) -> String { + // Do not forward the CDP error: a browser implementation may include the + // rejected cookie value in its diagnostic. + format!("failed to set cookie `{name}`") +} + +/// Installs host-only, root-scoped cookies before a page has an origin. +pub(crate) async fn set_browser_cookies( + browser: &Browser, + cookies: &[(String, String)], + url: &url::Url, +) -> Result<(), String> { + for (name, value) in cookies { + let cookie = host_cookie(name, value, url)?; + browser + .set_cookies(vec![cookie]) + .await + .map_err(|error| format_cookie_install_error(name, error))?; + } + Ok(()) +} + +/// Auto-detects a Chrome/Chromium executable. +/// +/// Searches `PATH` by common names first, then well-known per-OS install +/// locations (e.g. the macOS `.app` bundle, which is not on `PATH`). +fn find_chrome() -> Result { + if let Some(path) = CHROME_NAMES.iter().find_map(|name| which::which(name).ok()) { + return Ok(path); + } + if let Some(path) = well_known_chrome_paths() + .into_iter() + .find(|path| path.is_file()) + { + return Ok(path); + } + Err(format!( + "could not find Chrome/Chromium on PATH or in standard install locations (looked for: {})", + CHROME_NAMES.join(", ") + )) +} + +/// Well-known absolute Chrome/Chromium install locations for the host OS. +fn well_known_chrome_paths() -> Vec { + let mut paths = Vec::new(); + + #[cfg(target_os = "macos")] + { + const APPS: &[&str] = &[ + "Google Chrome.app/Contents/MacOS/Google Chrome", + "Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", + "Chromium.app/Contents/MacOS/Chromium", + ]; + for app in APPS { + paths.push(std::path::PathBuf::from(format!("/Applications/{app}"))); + if let Ok(home) = std::env::var("HOME") { + paths.push(std::path::PathBuf::from(format!( + "{home}/Applications/{app}" + ))); + } + } + } + + #[cfg(target_os = "linux")] + { + for path in [ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + "/snap/bin/chromium", + ] { + paths.push(std::path::PathBuf::from(path)); + } + } + + #[cfg(target_os = "windows")] + { + for path in [ + r"C:\Program Files\Google\Chrome\Application\chrome.exe", + r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe", + ] { + paths.push(std::path::PathBuf::from(path)); + } + } + + paths +} + +impl AuditCollector for BrowserCollector { + fn collect_page(&self, request: BrowserCollectRequest) -> Result { + self.collect_pages(std::slice::from_ref(&request)) + .into_iter() + .next() + .expect("should return one result for one browser request") + } + + fn collect_pages( + &self, + requests: &[BrowserCollectRequest], + ) -> Vec> { + if requests.is_empty() { + return Vec::new(); + } + // HTTP(S) scheme is enforced by the CLI value parser before we get here. + let chrome = match resolve_chrome(self.chrome.as_deref()) { + Ok(chrome) => chrome, + Err(error) => return vec![Err(error); requests.len()], + }; + let profile = match tempfile::tempdir() { + Ok(profile) => profile, + Err(error) => { + let error = format!("failed to create browser profile dir: {error}"); + return vec![Err(error); requests.len()]; + } + }; + + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let error = format!("failed to build browser runtime: {error}"); + return vec![Err(error); requests.len()]; + } + }; + + let settle = SettleConfig { + quiet: self.settle_quiet, + max: self.settle_max, + }; + + let accept_invalid_certs = self.accept_invalid_certs; + let headful = self.headful; + let assume_consent = self.assume_consent; + let proxy = self.proxy.clone(); + let browser_profile = self.profile; + let request_count = requests.len(); + let requests = requests.to_vec(); + let result = runtime.block_on(async move { + let options = BrowserSessionOptions { + chrome: &chrome, + profile_dir: profile.path(), + settle, + accept_invalid_certs, + headful, + assume_consent, + proxy: proxy.as_deref(), + profile: browser_profile, + }; + collect(requests, &options).await + }); + match result { + Ok(results) => results, + Err(error) => vec![Err(error); request_count], + } + } +} + +/// Drives a single page collection on the current-thread runtime. +async fn collect( + requests: Vec, + options: &BrowserSessionOptions<'_>, +) -> Result>, String> { + // chromiumoxide defaults to ignoring TLS errors. The audit sends + // operator-supplied session cookies and treats what it reads back as + // verification evidence, so a certificate-invalid impersonator could both + // harvest the session and fabricate the evidence. Validate certificates + // unless the operator explicitly opts out. + let (viewport, user_agent) = browser_profile(options.profile); + let config = build_browser_config(BrowserLaunchOptions { + chrome: options.chrome, + profile_dir: options.profile_dir, + headful: options.headful, + proxy: options.proxy, + accept_invalid_certs: options.accept_invalid_certs, + viewport, + user_agent, + })?; + + let (mut browser, mut handler) = Browser::launch(config) + .await + .map_err(|error| format!("failed to launch browser: {error}"))?; + + // Drive the CDP event loop for the duration of the session. + let handler_task = tokio::spawn(async move { while handler.next().await.is_some() {} }); + + let mut results = Vec::with_capacity(requests.len()); + for request in requests { + results.push( + collect_with_browser(&browser, request, options.settle, options.assume_consent).await, + ); + } + + // Best-effort teardown; ignore errors since we already have a result, but + // bound it so a Chrome that ignores `close` cannot hang the command. + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.close()).await; + let _ = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, browser.wait()).await; + handler_task.abort(); + + Ok(results) +} + +async fn collect_with_browser( + browser: &Browser, + request: BrowserCollectRequest, + settle_config: SettleConfig, + assume_consent: bool, +) -> Result { + set_browser_cookies(browser, &request.cookies, &request.url).await?; + + // Open a blank page first so init scripts are installed before the real + // document loads (evaluate-on-new-document applies to subsequent navigations). + let page = browser + .new_page("about:blank") + .await + .map_err(|error| format!("failed to open browser page: {error}"))?; + + let result = collect_open_page(&page, &request, settle_config, assume_consent).await; + let close_result = tokio::time::timeout(BROWSER_CLOSE_TIMEOUT, page.close()).await; + + match (result, close_result) { + (Err(error), _) => Err(error), + (Ok(mut collected), Err(_)) => { + collected.warnings.push(Warning { + code: "page_close_timeout".to_string(), + message: "timed out closing the browser tab after collection".to_string(), + }); + Ok(collected) + } + (Ok(mut collected), Ok(Err(error))) => { + collected.warnings.push(Warning { + code: "page_close_failed".to_string(), + message: format!("failed to close the browser tab after collection: {error}"), + }); + Ok(collected) + } + (Ok(collected), Ok(Ok(_))) => Ok(collected), + } +} + +/// Collects from an open tab. The caller owns tab teardown so every return path, +/// including an error from this function, closes the page before continuing. +async fn collect_open_page( + page: &Page, + request: &BrowserCollectRequest, + settle_config: SettleConfig, + assume_consent: bool, +) -> Result { + let mut warnings = Vec::new(); + + if assume_consent { + page.evaluate_on_new_document(CONSENT_STUB_SCRIPT) + .await + .map_err(|error| format!("failed to install consent init script: {error}"))?; + warnings.push(Warning { + code: "consent_stub_active".to_string(), + message: "audit consent APIs were stubbed; re-run with --no-assume-consent to observe the publisher CMP without substitution".to_string(), + }); + } + page.evaluate_on_new_document("performance.setResourceTimingBufferSize(100000)") + .await + .map_err(|error| format!("failed to increase resource timing buffer: {error}"))?; + + for script in &request.init_scripts { + page.evaluate_on_new_document(script.clone()) + .await + .map_err(|error| format!("failed to install init script: {error}"))?; + } + + tokio::time::timeout(NAVIGATION_TIMEOUT, page.goto(request.url.as_str())) + .await + .map_err(|_| format!("navigation to {} timed out", request.url))? + .map_err(|error| format!("failed to navigate to {}: {error}", request.url))?; + match tokio::time::timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation()).await { + Ok(Ok(_)) => {} + Ok(Err(error)) => warnings.push(Warning { + code: "navigation_wait_failed".to_string(), + message: format!( + "navigation load event could not be read ({error}); continuing with settled page evidence" + ), + }), + Err(_) => warnings.push(Warning { + code: "navigation_wait_timeout".to_string(), + message: format!( + "navigation did not fire its load event within {} seconds; continuing with settled page evidence", + NAVIGATION_TIMEOUT.as_secs() + ), + }), + } + + settle(page, settle_config, &mut warnings).await; + + if request.scroll { + if request.collect_ad_evidence { + // Snapshot evidence before scrolling so entries already present at + // initial load keep phase "load"; the store dedups first-seen, so + // the post-scroll scrape only adds genuinely scroll-phase entries. + if tokio::time::timeout( + CDP_OPERATION_TIMEOUT, + page.evaluate( + "(typeof window.__tsCollectAdTemplateEvidence === 'function' \ + && window.__tsCollectAdTemplateEvidence(), null)", + ), + ) + .await + .is_err() + { + warnings.push(Warning { + code: "ad_evidence_snapshot_timeout".to_string(), + message: "timed out snapshotting ad evidence before scroll".to_string(), + }); + } + } + scroll_page(page, &mut warnings).await; + settle(page, settle_config, &mut warnings).await; + } + + let final_url_text = tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.url()) + .await + .map_err(|_| "timed out reading final page URL".to_string())? + .map_err(|error| format!("failed to read final page URL: {error}"))? + .ok_or_else(|| "browser page URL was empty after navigation".to_string())?; + let final_url = url::Url::parse(&final_url_text).map_err(|error| { + format!("browser returned invalid final URL `{final_url_text}`: {error}") + })?; + let title = match tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.get_title()).await { + Ok(Ok(title)) => title.unwrap_or_default(), + Ok(Err(error)) => { + warnings.push(Warning { + code: "page_title_failed".to_string(), + message: format!("failed to read page title: {error}"), + }); + String::new() + } + Err(_) => { + warnings.push(Warning { + code: "page_title_timeout".to_string(), + message: "timed out reading page title".to_string(), + }); + String::new() + } + }; + let script_count = eval_usize(page, "document.querySelectorAll('script').length") + .await + .unwrap_or_else(|message| { + warnings.push(Warning { + code: "script_count_failed".to_string(), + message, + }); + 0 + }); + let resource_count = resource_count(page).await.unwrap_or_else(|message| { + warnings.push(Warning { + code: "resource_count_failed".to_string(), + message, + }); + 0 + }); + + if resource_count >= 250 { + warnings.push(Warning { + code: "resource_timing_heavy".to_string(), + message: format!("page recorded {resource_count} network resources"), + }); + } + + if let Ok(Ok(frames)) = tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.frames()).await + && frames.len() > 1 + { + warnings.push(Warning { + code: "child_frames_not_inspected".to_string(), + message: format!( + "ad-template evidence inspected only the main frame; {} child frame(s) were present", + frames.len() - 1 + ), + }); + } + + let ad_evidence = if request.collect_ad_evidence { + extract_ad_evidence(page, &mut warnings).await + } else { + None + }; + + Ok(CollectedPage { + final_url, + title, + script_count, + resource_count, + warnings, + ad_evidence, + }) +} + +/// Waits for the page network to go quiet after navigation or scroll. +/// +/// Polls the resource-entry count and returns once it stays unchanged for a +/// quiet window, or when the hard cap elapses — so ad-heavy pages finish loading +/// before evidence is read, without hanging on pages that never go idle. +async fn settle(page: &Page, config: SettleConfig, warnings: &mut Vec) { + let start = std::time::Instant::now(); + let mut last = None; + let mut quiet_since = None; + + loop { + if start.elapsed() >= config.max { + warnings.push(Warning { + code: "settle_timeout".to_string(), + message: "page did not settle before the configured maximum wait".to_string(), + }); + return; + } + + let ready_state = match eval_string(page, "document.readyState").await { + Ok(state) => state, + Err(message) => { + warnings.push(Warning { + code: "settle_read_failed".to_string(), + message, + }); + return; + } + }; + let current = match resource_count(page).await { + Ok(count) => count, + Err(message) => { + warnings.push(Warning { + code: "settle_read_failed".to_string(), + message, + }); + return; + } + }; + let ready = matches!(ready_state.as_str(), "interactive" | "complete"); + if ready && last == Some(current) { + let quiet_start = quiet_since.get_or_insert_with(std::time::Instant::now); + if quiet_start.elapsed() >= config.quiet { + return; + } + } else { + quiet_since = None; + } + last = Some(current); + + let remaining_max = config.max.saturating_sub(start.elapsed()); + let remaining_quiet = quiet_since + .map(|quiet_start| config.quiet.saturating_sub(quiet_start.elapsed())) + .unwrap_or(config.quiet); + let sleep_for = Duration::from_millis(SETTLE_POLL_MS) + .min(remaining_max) + .min(remaining_quiet.max(Duration::from_millis(1))); + tokio::time::sleep(sleep_for).await; + } +} + +/// Reads the number of resource timing entries observed so far. +async fn resource_count(page: &Page) -> Result { + eval_usize(page, "performance.getEntriesByType('resource').length").await +} + +/// Performs a deterministic stepped scroll to trigger lazy ad loading. +async fn scroll_page(page: &Page, warnings: &mut Vec) { + // Mark subsequent observations as scroll-phase for the collector. + eval_discard(page, "window.__tsScrollPhase = true", warnings).await; + for fraction in ["0.33", "0.66", "1"] { + let script = format!( + "window.scrollTo(0, Math.floor(Math.max(document.body.scrollHeight, \ + document.documentElement.scrollHeight) * {fraction}))" + ); + eval_discard(page, script, warnings).await; + tokio::time::sleep(Duration::from_millis(250)).await; + } + eval_discard(page, "window.scrollTo(0, 0)", warnings).await; +} + +async fn eval_discard(page: &Page, expression: impl Into, warnings: &mut Vec) { + match tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression.into())).await { + Ok(Ok(_)) => {} + Ok(Err(error)) => warnings.push(Warning { + code: "page_evaluation_failed".to_string(), + message: format!("browser page evaluation failed: {error}"), + }), + Err(_) => warnings.push(Warning { + code: "page_evaluation_timeout".to_string(), + message: "browser page evaluation timed out".to_string(), + }), + } +} + +async fn eval_usize(page: &Page, expression: &str) -> Result { + tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression)) + .await + .map_err(|_| format!("timed out evaluating `{expression}`"))? + .map_err(|error| format!("failed to evaluate `{expression}`: {error}"))? + .into_value::() + .map_err(|error| format!("failed to decode `{expression}`: {error}")) +} + +async fn eval_string(page: &Page, expression: &str) -> Result { + tokio::time::timeout(CDP_OPERATION_TIMEOUT, page.evaluate(expression)) + .await + .map_err(|_| format!("timed out evaluating `{expression}`"))? + .map_err(|error| format!("failed to evaluate `{expression}`: {error}"))? + .into_value::() + .map_err(|error| format!("failed to decode `{expression}`: {error}")) +} + +/// Reads and decodes `window.__tsAdTemplateEvidence`, warning (not failing) on a +/// decode error. +async fn extract_ad_evidence( + page: &Page, + warnings: &mut Vec, +) -> Option { + // Serialize and size-check in the page so a hostile publisher-controlled + // evidence object cannot force an unbounded CDP response and Rust decode. + let evaluation = tokio::time::timeout( + CDP_OPERATION_TIMEOUT, + page.evaluate(format!( + r#"(() => {{ + const evidence = typeof window.__tsCollectAdTemplateEvidence === 'function' + ? window.__tsCollectAdTemplateEvidence() + : (window.__tsAdTemplateEvidence || null) + if (evidence === null) return {{ kind: 'absent' }} + try {{ + const json = JSON.stringify(evidence) + const bytes = new TextEncoder().encode(json).byteLength + if (bytes > {MAX_EVIDENCE_PAYLOAD_BYTES}) return {{ kind: 'too_large' }} + return {{ kind: 'evidence', json }} + }} catch (error) {{ + return {{ + kind: 'serialization_failed', + message: String(error).slice(0, 512), + }} + }} + }})()"# + )), + ) + .await; + + let envelope = match evaluation { + Ok(Ok(result)) => match result.into_value::() { + Ok(envelope) => Some(envelope), + Err(error) => { + warnings.push(Warning { + code: "ad_evidence_decode_failed".to_string(), + message: format!("failed to decode ad-template evidence envelope: {error}"), + }); + return None; + } + }, + Ok(Err(error)) => { + warnings.push(Warning { + code: "ad_evidence_read_failed".to_string(), + message: format!("failed to read ad-template evidence: {error}"), + }); + return None; + } + Err(_) => { + warnings.push(Warning { + code: "ad_evidence_read_timeout".to_string(), + message: "timed out reading ad-template evidence".to_string(), + }); + return None; + } + }; + + match envelope { + Some(envelope) => decode_ad_evidence_envelope(envelope, warnings), + None => { + warnings.push(Warning { + code: "ad_evidence_absent".to_string(), + message: "no ad-template evidence was collected from the page".to_string(), + }); + None + } + } +} + +#[derive(Debug, serde::Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum EvidenceEnvelope { + Absent, + TooLarge, + Evidence { json: String }, + SerializationFailed { message: String }, +} + +fn decode_ad_evidence_envelope( + envelope: EvidenceEnvelope, + warnings: &mut Vec, +) -> Option { + match envelope { + EvidenceEnvelope::Absent => { + warnings.push(Warning { + code: "ad_evidence_absent".to_string(), + message: "no ad-template evidence was collected from the page".to_string(), + }); + None + } + EvidenceEnvelope::TooLarge => { + warnings.push(Warning { + code: "ad_evidence_too_large".to_string(), + message: format!( + "ad-template evidence exceeded the {MAX_EVIDENCE_PAYLOAD_BYTES}-byte limit" + ), + }); + None + } + EvidenceEnvelope::SerializationFailed { message } => { + warnings.push(Warning { + code: "ad_evidence_encode_failed".to_string(), + message: format!("failed to serialize ad-template evidence in the page: {message}"), + }); + None + } + EvidenceEnvelope::Evidence { json } => { + match serde_json::from_str::(&json) { + Ok(mut evidence) => { + // Defense in depth: the injected script caps these lists, but the + // page owns that store, so re-cap after decode. + evidence.dom_ids.truncate(MAX_EVIDENCE_ENTRIES); + evidence.gpt_slots.truncate(MAX_EVIDENCE_ENTRIES); + evidence.aps_calls.truncate(MAX_EVIDENCE_ENTRIES); + evidence.warnings.truncate(MAX_EVIDENCE_ENTRIES); + Some(evidence) + } + Err(error) => { + warnings.push(Warning { + code: "ad_evidence_decode_failed".to_string(), + message: format!("failed to decode ad-template evidence: {error}"), + }); + None + } + } + } + } +} + +/// Whether a Chrome/Chromium fixture is available for browser-backed tests. +/// +/// Skips optional local runs, but makes the scripted/CI contract fail loudly. +/// Shared with the generation collector's tests so the contract has one +/// definition. +#[cfg(test)] +pub(crate) fn browser_fixture_available() -> bool { + if resolve_chrome(None).is_ok() { + return true; + } + assert!( + std::env::var_os("TS_AUDIT_BROWSER_TESTS").is_none(), + "TS_AUDIT_BROWSER_TESTS requires Chrome/Chromium; set CHROME to its executable" + ); + false +} + +#[cfg(test)] +mod tests { + use std::io::{Read as _, Write as _}; + use std::net::TcpListener; + use std::sync::mpsc; + + use super::*; + use crate::commands::audit::collector::{ + AdTemplateCollectorConfig, build_ad_template_init_script, + }; + + #[test] + fn well_known_chrome_paths_are_known_for_this_os() { + // macOS/Linux/Windows each have candidate paths; guards the cfg branches. + assert!( + !well_known_chrome_paths().is_empty(), + "supported OSes should list candidate Chrome install paths" + ); + } + + #[test] + fn oversized_ad_evidence_is_an_explicit_warning() { + let mut warnings = Vec::new(); + let evidence = decode_ad_evidence_envelope(EvidenceEnvelope::TooLarge, &mut warnings); + + assert!(evidence.is_none()); + assert_eq!(warnings.len(), 1); + assert_eq!(warnings[0].code, "ad_evidence_too_large"); + } + + #[test] + fn supplied_cookie_is_host_only_and_root_scoped() { + let url = + url::Url::parse("https://publisher.example/news/story").expect("should parse test URL"); + let cookie = host_cookie("clearance", "token", &url).expect("should build cookie"); + + assert!(cookie.domain.is_none(), "host-only cookies omit Domain"); + assert_eq!(cookie.path.as_deref(), Some("/")); + assert_eq!( + cookie.url.as_deref(), + Some("https://publisher.example"), + "the origin scopes a host-only cookie before first navigation" + ); + assert_eq!(cookie.secure, Some(true), "HTTPS cookies must be Secure"); + } + + #[test] + fn cookie_install_error_identifies_name_without_a_value() { + let error = format_cookie_install_error( + "datadome", + "invalid cookie value operator-secret-cookie-value", + ); + + assert_eq!(error, "failed to set cookie `datadome`"); + assert!(!error.contains("operator-secret-cookie-value")); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn supplied_cookie_reaches_first_navigation() { + if !browser_fixture_available() { + return; + } + + let listener = TcpListener::bind("127.0.0.1:0").expect("should bind fixture server"); + let address = listener.local_addr().expect("should read fixture address"); + let (request_tx, request_rx) = mpsc::channel(); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("should accept browser request"); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .expect("should set fixture read timeout"); + let mut request = Vec::new(); + while !request.ends_with(b"\r\n\r\n") { + let mut chunk = [0_u8; 1024]; + let chunk_len = stream.read(&mut chunk).expect("should read HTTP request"); + assert!(chunk_len > 0, "request should contain complete headers"); + request.extend_from_slice(&chunk[..chunk_len]); + assert!( + request.len() <= 16 * 1024, + "request headers should be bounded" + ); + } + request_tx + .send(String::from_utf8_lossy(&request).into_owned()) + .expect("should send captured request"); + + let body = b"cookie fixture"; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .expect("should write fixture headers"); + stream.write_all(body).expect("should write fixture body"); + }); + + let collector = BrowserCollector { + settle_quiet: Duration::from_millis(100), + settle_max: Duration::from_secs(1), + ..BrowserCollector::new() + }; + collector + .collect_page(BrowserCollectRequest { + url: url::Url::parse(&format!("http://{address}/")) + .expect("should parse fixture URL"), + init_scripts: Vec::new(), + scroll: false, + collect_ad_evidence: false, + cookies: vec![("clearance".to_string(), "token".to_string())], + }) + .expect("cookie should be installed before first navigation"); + + let request = request_rx + .recv_timeout(Duration::from_secs(5)) + .expect("fixture should receive the first navigation"); + assert!( + request.lines().any(|line| { + line.split_once(':').is_some_and(|(name, value)| { + name.eq_ignore_ascii_case("cookie") + && value + .trim() + .split(';') + .any(|cookie| cookie.trim() == "clearance=token") + }) + }), + "first navigation should carry the supplied cookie; request was {request:?}" + ); + } + + /// A self-contained page that stubs just enough of GPT (no network) for the + /// collector to observe a defined slot via the wrapped `defineSlot` and the + /// `getSlots()` scrape. + const GPT_FIXTURE: &str = r#" + + + +
+ + + +"#; + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn collects_gpt_slot_from_local_fixture() { + if !browser_fixture_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: false, + collect_ad_evidence: true, + cookies: Vec::new(), + }) + .expect("should collect fixture page"); + + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence + .gpt_slots + .iter() + .any(|slot| slot.gam_unit_path == "/123/news/atf"), + "should capture the defined GPT slot" + ); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0"), + "should capture the configured-prefix DOM id" + ); + } + + #[test] + #[ignore = "requires local Chrome/Chromium; run through scripts/test-cli.sh"] + fn scroll_pass_keeps_initial_load_phase_for_load_time_evidence() { + if !browser_fixture_available() { + // Browser fixture test requires a local Chrome/Chromium; skipping. + return; + } + let mut fixture = tempfile::Builder::new() + .suffix(".html") + .tempfile() + .expect("should create fixture file"); + fixture + .write_all(GPT_FIXTURE.as_bytes()) + .expect("should write fixture"); + let url = url::Url::from_file_path(fixture.path()).expect("should build file url"); + + let script = build_ad_template_init_script(&AdTemplateCollectorConfig { + div_prefixes: vec!["ad-atf-".to_string()], + }) + .expect("should build init script"); + + let collector = BrowserCollector::new(); + let page = collector + .collect_page(BrowserCollectRequest { + url, + init_scripts: vec![script], + scroll: true, + collect_ad_evidence: true, + cookies: Vec::new(), + }) + .expect("should collect fixture page"); + + // The slot and DOM id exist at load time, so the pre-scroll snapshot + // must record them as initial-load even though a scroll pass ran. + let evidence = page.ad_evidence.expect("fixture should yield ad evidence"); + assert!( + evidence.dom_ids.iter().any(|dom| dom.dom_id == "ad-atf-0" + && dom.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad), + "load-time DOM id should keep phase initial_load under --scroll" + ); + assert!( + evidence.gpt_slots.iter().any(|slot| { + slot.gam_unit_path == "/123/news/atf" + && slot.phase == crate::ad_templates::compare::EvidencePhase::InitialLoad + }), + "load-time GPT slot should keep phase initial_load under --scroll" + ); + } +} diff --git a/crates/trusted-server-cli/src/commands/audit/browser_collector.rs b/crates/trusted-server-cli/src/commands/audit/browser_collector.rs deleted file mode 100644 index 87a2ccc2c..000000000 --- a/crates/trusted-server-cli/src/commands/audit/browser_collector.rs +++ /dev/null @@ -1,435 +0,0 @@ -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use chromiumoxide::ArcHttpRequest; -use chromiumoxide::browser::{Browser, BrowserConfig}; -use futures::StreamExt as _; -use serde::Deserialize; -use tempfile::TempDir; -use tokio::runtime::Builder; -use tokio::time::{sleep, timeout}; -use url::Url; -use which::which; - -use crate::commands::audit::collector::{ - AuditCollector, CollectedPage, CollectedRequest, CollectedScriptTag, -}; -use crate::error::{CliResult, report_error}; - -const SETTLE_QUIET_PERIOD: Duration = Duration::from_millis(750); -const SETTLE_POLL_INTERVAL: Duration = Duration::from_millis(250); -const SETTLE_MAX_WAIT: Duration = Duration::from_secs(6); -const NAVIGATION_TIMEOUT: Duration = Duration::from_secs(30); -const BROWSER_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); -const RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD: usize = 250; -const RESOURCE_TIMING_BUFFER_WARNING: &str = - "browser resource timing buffer reached its default size; some network assets may be missing"; - -#[derive(Default)] -pub(crate) struct BrowserAuditCollector; - -impl AuditCollector for BrowserAuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult { - let runtime = Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| { - report_error(format!( - "failed to build Tokio runtime for browser audit: {error}" - )) - })?; - - runtime.block_on(collect_page_via_browser_async(target_url)) - } -} - -async fn collect_page_via_browser_async(target_url: &Url) -> CliResult { - let chrome_executable = find_browser_executable()?; - let user_data_dir = TempDir::new().map_err(|error| { - report_error(format!( - "failed to create temporary browser profile for audit: {error}" - )) - })?; - let config = BrowserConfig::builder() - .chrome_executable(chrome_executable) - .user_data_dir(user_data_dir.path()) - .new_headless_mode() - .build() - .map_err(|error| { - report_error(format!( - "failed to build Chromium configuration for audit: {error}" - )) - })?; - - let (mut browser, mut handler) = Browser::launch(config).await.map_err(|error| { - report_error(format!( - "failed to launch Chrome/Chromium for audit: {error}" - )) - })?; - - let handler_task = tokio::spawn(async move { - while let Some(event) = handler.next().await { - if event.is_err() { - break; - } - } - }); - - let result = collect_page_from_browser(&mut browser, target_url).await; - - let close_result = timeout(BROWSER_CLOSE_TIMEOUT, browser.close()) - .await - .map_err(|_| report_error("timed out closing browser after audit")) - .and_then(|result| { - result.map_err(|error| { - report_error(format!("failed to close browser after audit: {error}")) - }) - }); - if close_result.is_err() { - handler_task.abort(); - } - let _ = handler_task.await; - - match (result, close_result) { - (Ok(collected), Ok(_)) => Ok(collected), - (Ok(_), Err(error)) | (Err(error), _) => Err(error), - } -} - -async fn collect_page_from_browser( - browser: &mut Browser, - target_url: &Url, -) -> CliResult { - let page = browser.new_page("about:blank").await.map_err(|error| { - report_error(format!("failed to create browser page for audit: {error}")) - })?; - - timeout(NAVIGATION_TIMEOUT, page.goto(target_url.as_str())) - .await - .map_err(|_| report_error(format!("timed out navigating to `{target_url}`")))? - .map_err(|error| report_error(format!("failed to navigate to `{target_url}`: {error}")))?; - - let navigation_response = timeout(NAVIGATION_TIMEOUT, page.wait_for_navigation_response()) - .await - .map_err(|_| { - report_error(format!( - "timed out waiting for main document navigation response from `{target_url}`" - )) - })? - .map_err(|error| { - report_error(format!( - "failed to read main document navigation response: {error}" - )) - })?; - - let mut warnings = Vec::new(); - if let Some(warning) = validate_navigation_response(navigation_response)? { - warnings.push(warning); - } - if !wait_for_page_settle(&page).await? { - warnings.push( - "browser audit timed out while waiting for the page to settle; results may be partial" - .to_string(), - ); - } - - let final_url = page - .url() - .await - .map_err(|error| report_error(format!("failed to read final page URL: {error}")))? - .ok_or_else(|| report_error("browser page URL was empty after navigation"))?; - let page_title = page - .get_title() - .await - .map_err(|error| report_error(format!("failed to read page title: {error}")))?; - let html = page - .content() - .await - .map_err(|error| report_error(format!("failed to read rendered page HTML: {error}")))?; - - let script_tags: Vec = page - .evaluate( - r#"() => Array.from(document.scripts).map((script) => ({ - src: script.src || null, - inline_text: script.src ? null : (script.textContent || null), - }))"#, - ) - .await - .map_err(|error| report_error(format!("failed to read rendered script tags: {error}")))? - .into_value() - .map_err(|error| { - report_error(format!( - "failed to decode rendered script tag data: {error}" - )) - })?; - - let network_requests: Vec = page - .evaluate( - r#"() => performance.getEntriesByType('resource').map((entry) => ({ - url: entry.name, - initiator_type: entry.initiatorType || null, - }))"#, - ) - .await - .map_err(|error| { - report_error(format!( - "failed to read browser performance resource entries: {error}" - )) - })? - .into_value() - .map_err(|error| { - report_error(format!( - "failed to decode browser performance resource data: {error}" - )) - })?; - - if let Some(warning) = resource_timing_buffer_warning(network_requests.len()) { - warnings.push(warning.to_string()); - } - - Ok(CollectedPage { - requested_url: target_url.to_string(), - final_url, - page_title: page_title.filter(|title| !title.trim().is_empty()), - html, - script_tags: script_tags - .into_iter() - .map(|script| CollectedScriptTag { - src: script.src, - inline_text: script.inline_text.filter(|text| !text.trim().is_empty()), - }) - .collect(), - network_requests: network_requests - .into_iter() - .map(|entry| CollectedRequest { - url: entry.url, - resource_type: entry.initiator_type, - }) - .collect(), - warnings, - }) -} - -async fn wait_for_page_settle(page: &chromiumoxide::Page) -> CliResult { - let mut elapsed = Duration::ZERO; - let mut previous_count = None; - let mut stable_for = Duration::ZERO; - - while elapsed < SETTLE_MAX_WAIT { - let ready_state: String = page - .evaluate("document.readyState") - .await - .map_err(|error| report_error(format!("failed to read document ready state: {error}")))? - .into_value() - .map_err(|error| { - report_error(format!("failed to decode document ready state: {error}")) - })?; - let resource_count: usize = page - .evaluate("performance.getEntriesByType('resource').length") - .await - .map_err(|error| report_error(format!("failed to read resource count: {error}")))? - .into_value() - .map_err(|error| report_error(format!("failed to decode resource count: {error}")))?; - - if ready_state == "complete" { - if previous_count == Some(resource_count) { - stable_for += SETTLE_POLL_INTERVAL; - } else { - stable_for = Duration::ZERO; - } - - if stable_for >= SETTLE_QUIET_PERIOD { - return Ok(true); - } - } - - previous_count = Some(resource_count); - sleep(SETTLE_POLL_INTERVAL).await; - elapsed += SETTLE_POLL_INTERVAL; - } - - Ok(false) -} - -fn validate_navigation_response(navigation_response: ArcHttpRequest) -> CliResult> { - let request = navigation_response - .ok_or_else(|| report_error("browser audit did not capture the main document response"))?; - - if let Some(failure_text) = &request.failure_text { - return Err(report_error(format!( - "main document request failed: {failure_text}" - ))); - } - - let response = request.response.as_ref().ok_or_else(|| { - report_error("browser audit did not capture the main document HTTP response") - })?; - - if is_successful_navigation_status(response.status) { - return Ok(None); - } - - Ok(Some(format!( - "audit request returned HTTP {} {} for `{}`; results may be partial", - response.status, response.status_text, response.url - ))) -} - -fn is_successful_navigation_status(status: i64) -> bool { - (200..400).contains(&status) -} - -fn resource_timing_buffer_warning(resource_count: usize) -> Option<&'static str> { - (resource_count >= RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD) - .then_some(RESOURCE_TIMING_BUFFER_WARNING) -} - -fn find_browser_executable() -> CliResult { - for candidate in browser_executable_path_candidates() { - if let Ok(path) = which(candidate) { - return Ok(path); - } - } - - for candidate in browser_executable_fallbacks() { - let candidate_path = Path::new(candidate); - if candidate_path.is_file() { - return Ok(candidate_path.to_path_buf()); - } - } - - Err(report_error( - "Chrome/Chromium was not found on PATH or in the standard local install locations checked by `ts audit`. Install a local Chrome or Chromium binary before running `ts audit`.", - )) -} - -fn browser_executable_path_candidates() -> &'static [&'static str] { - &[ - "google-chrome", - "google-chrome-stable", - "chromium", - "chromium-browser", - "chrome", - "Google Chrome", - "Google Chrome for Testing", - ] -} - -fn browser_executable_fallbacks() -> &'static [&'static str] { - #[cfg(target_os = "macos")] - { - &[ - "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", - "/Applications/Chromium.app/Contents/MacOS/Chromium", - "/Applications/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing", - ] - } - - #[cfg(target_os = "linux")] - { - &[ - "/usr/bin/google-chrome", - "/usr/bin/google-chrome-stable", - "/usr/bin/chromium", - "/usr/bin/chromium-browser", - "/snap/bin/chromium", - ] - } - - #[cfg(not(any(target_os = "macos", target_os = "linux")))] - { - &[] - } -} - -#[derive(Debug, Deserialize)] -struct BrowserScriptTag { - src: Option, - inline_text: Option, -} - -#[derive(Debug, Deserialize)] -struct BrowserPerformanceEntry { - url: String, - initiator_type: Option, -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use chromiumoxide::cdp::browser_protocol::network::{Headers, RequestId, Response}; - use chromiumoxide::cdp::browser_protocol::security::SecurityState; - use chromiumoxide::handler::http::HttpRequest; - - use super::*; - - #[test] - fn successful_navigation_status_allows_redirects_but_rejects_errors() { - assert!(is_successful_navigation_status(200)); - assert!(is_successful_navigation_status(302)); - assert!(is_successful_navigation_status(399)); - assert!(!is_successful_navigation_status(199)); - assert!(!is_successful_navigation_status(400)); - assert!(!is_successful_navigation_status(500)); - } - - #[test] - fn navigation_response_returns_warning_for_http_error_status() { - let warning = - validate_navigation_response(navigation_response_with_status(403, "Forbidden")) - .expect("should validate navigation response") - .expect("should return warning for HTTP error status"); - - assert_eq!( - warning, - "audit request returned HTTP 403 Forbidden for `https://example.com/`; results may be partial", - "should warn and continue when the main document returns an HTTP error" - ); - } - - #[test] - fn resource_timing_buffer_warning_starts_at_threshold() { - assert_eq!( - resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD - 1), - None, - "should not warn before the resource timing buffer threshold" - ); - assert_eq!( - resource_timing_buffer_warning(RESOURCE_TIMING_BUFFER_WARNING_THRESHOLD), - Some(RESOURCE_TIMING_BUFFER_WARNING), - "should warn when the resource timing buffer reaches the threshold" - ); - } - - #[test] - fn browser_path_candidates_include_common_names() { - let candidates = browser_executable_path_candidates(); - - assert!(candidates.contains(&"google-chrome")); - assert!(candidates.contains(&"chromium")); - assert!(candidates.contains(&"Google Chrome for Testing")); - } - - fn navigation_response_with_status(status: i64, status_text: &str) -> ArcHttpRequest { - let mut request = - HttpRequest::new(RequestId::new("request-1"), None, None, false, Vec::new()); - request.response = Some( - Response::builder() - .url("https://example.com/") - .status(status) - .status_text(status_text) - .headers(Headers::default()) - .mime_type("text/html") - .charset("utf-8") - .connection_reused(false) - .connection_id(1.0) - .encoded_data_length(0.0) - .security_state(SecurityState::Secure) - .build() - .expect("should build navigation response"), - ); - - Some(Arc::new(request)) - } -} diff --git a/crates/trusted-server-cli/src/commands/audit/collector.rs b/crates/trusted-server-cli/src/commands/audit/collector.rs index 314ae54fc..6ab427b2c 100644 --- a/crates/trusted-server-cli/src/commands/audit/collector.rs +++ b/crates/trusted-server-cli/src/commands/audit/collector.rs @@ -1,41 +1,327 @@ -use serde::{Deserialize, Serialize}; -use url::Url; +//! Collector abstraction shared by the generic page audit and the ad-template +//! verifier. +//! +//! Decoupling collection behind [`AuditCollector`] lets the verifier orchestration +//! (Task 9) be tested with an in-memory fake collector, with no Chrome dependency. -use crate::error::CliResult; +use std::path::PathBuf; -pub(crate) trait AuditCollector { - fn collect_page(&self, target_url: &Url) -> CliResult; +use clap::{Args, ValueEnum}; + +use crate::ad_templates::compare::BrowserAdEvidence; + +/// Operator-tunable browser options shared by `ts audit page` and +/// `ts audit ad-templates verify`. +/// +/// These are audit-tool knobs, not publisher runtime config, so they live on the +/// CLI (flags / `CHROME` env) rather than in `trusted-server.toml`. +#[derive(Debug, Clone, Args)] +pub struct BrowserOpts { + /// Path to the Chrome/Chromium executable. Falls back to `$CHROME`, then + /// auto-detection on `PATH` and standard install locations. + #[arg(long)] + pub chrome: Option, + /// Browser device profile used for viewport and user-agent emulation. + #[arg(long = "browser-profile", value_enum, default_value_t = BrowserProfile::Desktop)] + pub profile: BrowserProfile, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long)] + pub headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long)] + pub no_assume_consent: bool, + /// Route the browser through this proxy, as `host:port` or a full URL. + #[arg(long, value_name = "HOST:PORT")] + pub browser_proxy: Option, + /// Quiet window in milliseconds (no new network resources) that marks the + /// page settled. + #[arg(long, default_value_t = 750)] + pub settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg(long, default_value_t = 10_000)] + pub settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + /// + /// DANGEROUS: the audit sends any `--cookie` session to the origin and + /// treats what it reads back as verification evidence, so an invalid + /// certificate could mean an impersonator is harvesting the session and + /// fabricating the evidence. Use only against a host you control with a + /// known self-signed certificate. + #[arg(long)] + pub danger_accept_invalid_certs: bool, +} + +/// Browser options for generation, whose device selection is controlled by +/// `--profiles` rather than the verifier's singular `--browser-profile`. +#[derive(Debug, Clone, Args)] +pub struct GenerateBrowserOpts { + /// Path to the Chrome/Chromium executable. Falls back to `$CHROME`, then auto-detection. + #[arg(long)] + pub chrome: Option, + /// Run a visible browser instead of Chrome's new headless mode. + #[arg(long)] + pub headful: bool, + /// Do not answer the standard IAB consent APIs for the fresh audit profile. + #[arg(long)] + pub no_assume_consent: bool, + /// Route the browser through this proxy, as `host:port` or a full URL. + #[arg(long, value_name = "HOST:PORT")] + pub browser_proxy: Option, + /// Quiet window in milliseconds that marks the page settled. + #[arg(long, default_value_t = 750)] + pub settle_quiet_ms: u64, + /// Hard cap in milliseconds on waiting for the page to settle. + #[arg(long, default_value_t = 10_000)] + pub settle_max_ms: u64, + /// Navigate to origins whose TLS certificate does not validate. + /// + /// DANGEROUS: the audit sends any `--cookie` session to the origin and + /// treats what it reads back as the evidence it writes config from, so an + /// invalid certificate could mean an impersonator is harvesting the session + /// and fabricating the evidence. Use only against a host you control with a + /// known self-signed certificate. + #[arg(long)] + pub danger_accept_invalid_certs: bool, +} + +/// Defaults mirroring the `#[arg(default_value_t)]` values above, so a path that +/// builds these options in code (the legacy `ts audit ` form) behaves like +/// the parsed command. +impl Default for GenerateBrowserOpts { + fn default() -> Self { + Self { + chrome: None, + headful: false, + no_assume_consent: false, + browser_proxy: None, + settle_quiet_ms: 750, + settle_max_ms: 10_000, + danger_accept_invalid_certs: false, + } + } +} + +impl GenerateBrowserOpts { + /// Validates relationships between independently parsed browser flags. + pub fn validate(&self) -> Result<(), String> { + validate_settle_window(self.settle_quiet_ms, self.settle_max_ms) + } +} + +/// Browser device profile shared by page audits and ad-template verification. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, ValueEnum)] +pub enum BrowserProfile { + /// Desktop Chrome at 1280×800. + #[default] + Desktop, + /// Mobile-sized viewport with a mobile user agent. + Mobile, +} + +impl BrowserOpts { + /// Validates relationships between independently parsed browser flags. + pub fn validate(&self) -> Result<(), String> { + validate_settle_window(self.settle_quiet_ms, self.settle_max_ms) + } +} + +fn validate_settle_window(quiet_ms: u64, max_ms: u64) -> Result<(), String> { + if quiet_ms > max_ms { + return Err(format!( + "--settle-quiet-ms ({quiet_ms}) cannot exceed --settle-max-ms ({max_ms})" + )); + } + Ok(()) +} + +/// A request to collect a single page. +#[derive(Debug, Clone)] +pub struct BrowserCollectRequest { + /// The URL to navigate to. + pub url: url::Url, + /// Pre-navigation init scripts (evaluate-on-new-document). Empty for a plain + /// page audit; the ad-template verifier supplies the read-only collector here. + pub init_scripts: Vec, + /// Whether to perform the deterministic scroll pass after settle. + pub scroll: bool, + /// Whether to extract `window.__tsAdTemplateEvidence` after settle/scroll. + pub collect_ad_evidence: bool, + /// Operator-supplied `(name, value)` cookies set on the browser context + /// before navigation, scoped to the request URL. Used to carry an existing + /// authenticated session (e.g. a valid bot-protection clearance cookie) so + /// the origin serves the real page instead of a challenge. The collector + /// only sends these; it never reads cookies back. + pub cookies: Vec<(String, String)>, +} + +/// The result of collecting a single page. +#[derive(Debug, Clone)] +pub struct CollectedPage { + /// The final URL after redirects. + pub final_url: url::Url, + /// The page title. + pub title: String, + /// Number of `